[ 544.865347] env[63593]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'linux_bridge' {{(pid=63593) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 544.865753] env[63593]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'noop' {{(pid=63593) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 544.865853] env[63593]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'ovs' {{(pid=63593) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 544.866228] env[63593]: INFO os_vif [-] Loaded VIF plugins: linux_bridge, noop, ovs [ 544.960543] env[63593]: DEBUG oslo_concurrency.processutils [-] Running cmd (subprocess): grep -F node.session.scan /sbin/iscsiadm {{(pid=63593) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:390}} [ 544.970828] env[63593]: DEBUG oslo_concurrency.processutils [-] CMD "grep -F node.session.scan /sbin/iscsiadm" returned: 0 in 0.010s {{(pid=63593) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:428}} [ 545.576020] env[63593]: INFO nova.virt.driver [None req-be1a0175-8b8d-4c41-b181-22189c4d743f None None] Loading compute driver 'vmwareapi.VMwareVCDriver' [ 545.645195] env[63593]: DEBUG oslo_concurrency.lockutils [-] Acquiring lock "oslo_vmware_api_lock" by "oslo_vmware.api.VMwareAPISession._create_session" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 545.645580] env[63593]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" acquired by "oslo_vmware.api.VMwareAPISession._create_session" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 545.645792] env[63593]: DEBUG oslo_vmware.service [-] Creating suds client with soap_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk' and wsdl_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk/vimService.wsdl' {{(pid=63593) __init__ /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:242}} [ 548.670894] env[63593]: DEBUG oslo_vmware.service [-] Invoking ServiceInstance.RetrieveServiceContent with opID=oslo.vmware-2dc08cf6-95d8-4868-ab05-cd29a1848c19 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.686636] env[63593]: DEBUG oslo_vmware.api [-] Logging into host: vc1.osci.c.eu-de-1.cloud.sap. {{(pid=63593) _create_session /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:242}} [ 548.686842] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.Login with opID=oslo.vmware-7dacd873-c63a-47f5-8eb9-09b25f57b610 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.711552] env[63593]: INFO oslo_vmware.api [-] Successfully established new session; session ID is 3f9f1. [ 548.711738] env[63593]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" "released" by "oslo_vmware.api.VMwareAPISession._create_session" :: held 3.066s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 548.712203] env[63593]: INFO nova.virt.vmwareapi.driver [None req-be1a0175-8b8d-4c41-b181-22189c4d743f None None] VMware vCenter version: 7.0.3 [ 548.715593] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb9aed63-682f-43c5-ad59-92be07a3d51b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.736572] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd6bb39c-15a6-453c-b593-701196d6fd1e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.742179] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db533df0-a031-4e19-b739-716db863074c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.748964] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4d48718-788d-4ffb-a92e-830e4461d6a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.761564] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39f150ab-ca24-4f5f-80ab-f0fdd61c27b2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.767235] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-681a595b-7866-4708-b1e1-12bc9a040e69 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.796632] env[63593]: DEBUG oslo_vmware.service [-] Invoking ExtensionManager.FindExtension with opID=oslo.vmware-31ee9fbc-e178-4924-aca2-04bb82a3266b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.801308] env[63593]: DEBUG nova.virt.vmwareapi.driver [None req-be1a0175-8b8d-4c41-b181-22189c4d743f None None] Extension org.openstack.compute already exists. {{(pid=63593) _register_openstack_extension /opt/stack/nova/nova/virt/vmwareapi/driver.py:224}} [ 548.804032] env[63593]: INFO nova.compute.provider_config [None req-be1a0175-8b8d-4c41-b181-22189c4d743f None None] No provider configs found in /etc/nova/provider_config/. If files are present, ensure the Nova process has access. [ 549.307015] env[63593]: DEBUG nova.context [None req-be1a0175-8b8d-4c41-b181-22189c4d743f None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),5b09cd6f-575b-4f71-84c2-9276ccdcf252(cell1) {{(pid=63593) load_cells /opt/stack/nova/nova/context.py:464}} [ 549.310053] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 549.310302] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 549.310980] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 549.311430] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Acquiring lock "5b09cd6f-575b-4f71-84c2-9276ccdcf252" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 549.311618] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Lock "5b09cd6f-575b-4f71-84c2-9276ccdcf252" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 549.312657] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Lock "5b09cd6f-575b-4f71-84c2-9276ccdcf252" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 549.333169] env[63593]: INFO dbcounter [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Registered counter for database nova_cell0 [ 549.341346] env[63593]: INFO dbcounter [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Registered counter for database nova_cell1 [ 549.344558] env[63593]: DEBUG oslo_db.sqlalchemy.engines [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=63593) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 549.345178] env[63593]: DEBUG oslo_db.sqlalchemy.engines [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=63593) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 549.349835] env[63593]: ERROR nova.db.main.api [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 549.349835] env[63593]: result = function(*args, **kwargs) [ 549.349835] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 549.349835] env[63593]: return func(*args, **kwargs) [ 549.349835] env[63593]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 549.349835] env[63593]: result = fn(*args, **kwargs) [ 549.349835] env[63593]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 549.349835] env[63593]: return f(*args, **kwargs) [ 549.349835] env[63593]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 549.349835] env[63593]: return db.service_get_minimum_version(context, binaries) [ 549.349835] env[63593]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 549.349835] env[63593]: _check_db_access() [ 549.349835] env[63593]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 549.349835] env[63593]: stacktrace = ''.join(traceback.format_stack()) [ 549.349835] env[63593]: [ 549.350858] env[63593]: ERROR nova.db.main.api [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 549.350858] env[63593]: result = function(*args, **kwargs) [ 549.350858] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 549.350858] env[63593]: return func(*args, **kwargs) [ 549.350858] env[63593]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 549.350858] env[63593]: result = fn(*args, **kwargs) [ 549.350858] env[63593]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 549.350858] env[63593]: return f(*args, **kwargs) [ 549.350858] env[63593]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 549.350858] env[63593]: return db.service_get_minimum_version(context, binaries) [ 549.350858] env[63593]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 549.350858] env[63593]: _check_db_access() [ 549.350858] env[63593]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 549.350858] env[63593]: stacktrace = ''.join(traceback.format_stack()) [ 549.350858] env[63593]: [ 549.351336] env[63593]: WARNING nova.objects.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Failed to get minimum service version for cell 00000000-0000-0000-0000-000000000000 [ 549.351403] env[63593]: WARNING nova.objects.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Failed to get minimum service version for cell 5b09cd6f-575b-4f71-84c2-9276ccdcf252 [ 549.351849] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Acquiring lock "singleton_lock" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 549.352035] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Acquired lock "singleton_lock" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 549.352294] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Releasing lock "singleton_lock" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 549.352615] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Full set of CONF: {{(pid=63593) _wait_for_exit_or_signal /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:363}} [ 549.352755] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ******************************************************************************** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2806}} [ 549.352898] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Configuration options gathered from: {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2807}} [ 549.353055] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] command line args: ['--config-file', '/etc/nova/nova.conf', '--config-file', '/etc/nova/nova-cpu-common.conf', '--config-file', '/etc/nova/nova-cpu-1.conf'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2808}} [ 549.353223] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] config files: ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2809}} [ 549.353349] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ================================================================================ {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2811}} [ 549.353563] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] allow_resize_to_same_host = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.353732] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] arq_binding_timeout = 300 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.353863] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] backdoor_port = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.353987] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] backdoor_socket = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.354160] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] block_device_allocate_retries = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.354322] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] block_device_allocate_retries_interval = 3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.354488] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cert = self.pem {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.354693] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute_driver = vmwareapi.VMwareVCDriver {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.354895] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute_monitors = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.355086] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] config_dir = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.355262] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] config_drive_format = iso9660 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.355396] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] config_file = ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.355588] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] config_source = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.355762] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] console_host = devstack {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.355925] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] control_exchange = nova {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.356100] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cpu_allocation_ratio = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.356263] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] daemon = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.356429] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] debug = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.356583] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] default_access_ip_network_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.356749] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] default_availability_zone = nova {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.356900] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] default_ephemeral_format = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.357068] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] default_green_pool_size = 1000 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.357307] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.357470] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] default_schedule_zone = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.357627] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] disk_allocation_ratio = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.357810] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] enable_new_services = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.358007] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] enabled_apis = ['osapi_compute'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.358181] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] enabled_ssl_apis = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.358341] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] flat_injected = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.358498] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] force_config_drive = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.358655] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] force_raw_images = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.358827] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] graceful_shutdown_timeout = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.358985] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] heal_instance_info_cache_interval = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.359216] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] host = cpu-1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.359447] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] initial_cpu_allocation_ratio = 4.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.359626] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] initial_disk_allocation_ratio = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.359871] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] initial_ram_allocation_ratio = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.360144] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.360317] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] instance_build_timeout = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.360480] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] instance_delete_interval = 300 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.360650] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] instance_format = [instance: %(uuid)s] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.360820] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] instance_name_template = instance-%08x {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.360982] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] instance_usage_audit = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.361168] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] instance_usage_audit_period = month {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.361336] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.361531] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] instances_path = /opt/stack/data/nova/instances {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.361711] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] internal_service_availability_zone = internal {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.361869] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] key = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.362084] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] live_migration_retry_count = 30 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.362307] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] log_color = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.362575] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] log_config_append = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.362785] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.362961] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] log_dir = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.363138] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] log_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.363272] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] log_options = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.363436] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] log_rotate_interval = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.363604] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] log_rotate_interval_type = days {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.363772] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] log_rotation_type = none {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.363900] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.364035] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.364210] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.364374] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.364503] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.364686] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] long_rpc_timeout = 1800 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.364851] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] max_concurrent_builds = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.365014] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] max_concurrent_live_migrations = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.365181] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] max_concurrent_snapshots = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.365338] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] max_local_block_devices = 3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.365498] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] max_logfile_count = 30 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.365678] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] max_logfile_size_mb = 200 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.365847] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] maximum_instance_delete_attempts = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.366027] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] metadata_listen = 0.0.0.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.366199] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] metadata_listen_port = 8775 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.366366] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] metadata_workers = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.366526] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] migrate_max_retries = -1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.366691] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] mkisofs_cmd = genisoimage {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.366896] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] my_block_storage_ip = 10.180.1.21 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.367038] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] my_ip = 10.180.1.21 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.367206] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] network_allocate_retries = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.367384] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.367552] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] osapi_compute_listen = 0.0.0.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.367718] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] osapi_compute_listen_port = 8774 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.367881] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] osapi_compute_unique_server_name_scope = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.368060] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] osapi_compute_workers = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.368227] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] password_length = 12 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.368386] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] periodic_enable = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.368544] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] periodic_fuzzy_delay = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.368738] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] pointer_model = usbtablet {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.368910] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] preallocate_images = none {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.369085] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] publish_errors = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.369217] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] pybasedir = /opt/stack/nova {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.369373] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ram_allocation_ratio = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.369532] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] rate_limit_burst = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.369698] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] rate_limit_except_level = CRITICAL {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.369856] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] rate_limit_interval = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.370023] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] reboot_timeout = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.370184] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] reclaim_instance_interval = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.370339] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] record = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.370503] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] reimage_timeout_per_gb = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.370667] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] report_interval = 120 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.370826] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] rescue_timeout = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.370983] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] reserved_host_cpus = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.371153] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] reserved_host_disk_mb = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.371311] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] reserved_host_memory_mb = 512 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.371467] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] reserved_huge_pages = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.371632] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] resize_confirm_window = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.371813] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] resize_fs_using_block_device = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.371974] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] resume_guests_state_on_host_boot = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.372156] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.372318] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] rpc_response_timeout = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.372474] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] run_external_periodic_tasks = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.372639] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] running_deleted_instance_action = reap {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.372798] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] running_deleted_instance_poll_interval = 1800 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.372954] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] running_deleted_instance_timeout = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.373123] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler_instance_sync_interval = 120 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.373289] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_down_time = 720 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.373454] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] servicegroup_driver = db {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.373609] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] shell_completion = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.373765] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] shelved_offload_time = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.373921] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] shelved_poll_interval = 3600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.374095] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] shutdown_timeout = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.374256] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] source_is_ipv6 = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.374412] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ssl_only = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.374693] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] state_path = /opt/stack/data/n-cpu-1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.374874] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] sync_power_state_interval = 600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.375049] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] sync_power_state_pool_size = 1000 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.375219] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] syslog_log_facility = LOG_USER {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.375376] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] tempdir = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.375534] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] timeout_nbd = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.375702] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] transport_url = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.375859] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] update_resources_interval = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.376031] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] use_cow_images = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.376198] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] use_eventlog = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.376357] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] use_journal = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.376512] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] use_json = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.376671] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] use_rootwrap_daemon = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.376827] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] use_stderr = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.376982] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] use_syslog = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.377152] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vcpu_pin_set = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.377320] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plugging_is_fatal = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.377486] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plugging_timeout = 300 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.377650] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] virt_mkfs = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.377841] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] volume_usage_poll_interval = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.378008] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] watch_log_file = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.378185] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] web = /usr/share/spice-html5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2819}} [ 549.378371] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_concurrency.disable_process_locking = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.378654] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_concurrency.lock_path = /opt/stack/data/n-cpu-1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.378838] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.379020] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.379194] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_metrics.metrics_process_name = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.379364] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.379529] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.379712] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.auth_strategy = keystone {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.379876] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.compute_link_prefix = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.380065] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.380242] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.dhcp_domain = novalocal {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.380411] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.enable_instance_password = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.380575] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.glance_link_prefix = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.380763] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.380950] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.instance_list_cells_batch_strategy = distributed {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.381132] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.instance_list_per_project_cells = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.381300] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.list_records_by_skipping_down_cells = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.381464] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.local_metadata_per_cell = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.381635] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.max_limit = 1000 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.381805] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.metadata_cache_expiration = 15 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.381979] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.neutron_default_tenant_id = default {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.382163] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.use_neutron_default_nets = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.382332] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.382497] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.vendordata_dynamic_failure_fatal = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.382660] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.382847] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.vendordata_dynamic_ssl_certfile = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.383032] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.vendordata_dynamic_targets = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.383199] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.vendordata_jsonfile_path = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.383383] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api.vendordata_providers = ['StaticJSON'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.383576] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.backend = dogpile.cache.memcached {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.383767] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.backend_argument = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.383958] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.config_prefix = cache.oslo {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.384152] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.dead_timeout = 60.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.384321] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.debug_cache_backend = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.384487] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.enable_retry_client = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.384686] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.enable_socket_keepalive = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.384868] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.enabled = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.385047] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.enforce_fips_mode = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.385219] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.expiration_time = 600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.385385] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.hashclient_retry_attempts = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.385555] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.hashclient_retry_delay = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.385725] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_dead_retry = 300 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.385886] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_password = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.386071] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.386244] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.386411] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_pool_maxsize = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.386577] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_pool_unused_timeout = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.386750] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_sasl_enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.386961] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_servers = ['localhost:11211'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.387151] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_socket_timeout = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.387318] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.memcache_username = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.387488] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.proxies = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.387655] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.redis_db = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.387820] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.redis_password = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.387990] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.redis_sentinel_service_name = mymaster {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.388185] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.redis_sentinels = ['localhost:26379'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.388356] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.redis_server = localhost:6379 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.388524] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.redis_socket_timeout = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.388685] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.redis_username = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.388847] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.retry_attempts = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.389022] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.retry_delay = 0.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.389189] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.socket_keepalive_count = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.389351] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.socket_keepalive_idle = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.389512] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.socket_keepalive_interval = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.389671] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.tls_allowed_ciphers = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.389848] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.tls_cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.390037] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.tls_certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.390209] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.tls_enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.390371] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cache.tls_keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.390542] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.auth_section = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.390720] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.auth_type = password {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.390878] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.391065] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.catalog_info = volumev3::publicURL {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.391230] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.391392] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.391554] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.cross_az_attach = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.391750] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.debug = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.391922] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.endpoint_template = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.392099] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.http_retries = 3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.392268] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.392444] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.392635] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.os_region_name = RegionOne {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.392811] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.392993] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cinder.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.393187] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.393351] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.cpu_dedicated_set = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.393511] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.cpu_shared_set = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.393677] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.image_type_exclude_list = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.393841] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.live_migration_wait_for_vif_plug = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.394013] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.max_concurrent_disk_ops = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.394185] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.max_disk_devices_to_attach = -1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.394391] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.394635] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.394816] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.resource_provider_association_refresh = 300 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.394981] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.shutdown_retry_interval = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.395183] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] compute.vmdk_allowed_types = ['streamOptimized', 'monolithicSparse'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.395363] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] conductor.workers = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.395544] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] console.allowed_origins = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.395708] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] console.ssl_ciphers = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.395880] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] console.ssl_minimum_version = default {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.396067] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] consoleauth.enforce_session_timeout = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.396242] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] consoleauth.token_ttl = 600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.396410] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.396571] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.396735] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.396894] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.connect_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.397063] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.connect_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.397224] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.endpoint_override = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.397384] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.397538] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.397696] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.max_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.397851] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.min_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.398009] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.region_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.398181] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.retriable_status_codes = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.398332] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.service_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.398497] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.service_type = accelerator {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.398658] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.398815] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.status_code_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.398970] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.status_code_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.399140] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.399322] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.399486] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] cyborg.version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.399667] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.backend = sqlalchemy {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.399839] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.connection = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.400013] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.connection_debug = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.400188] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.connection_parameters = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.400354] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.connection_recycle_time = 3600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.400517] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.connection_trace = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.400679] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.db_inc_retry_interval = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.400843] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.db_max_retries = 20 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.401011] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.db_max_retry_interval = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.401182] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.db_retry_interval = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.401344] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.max_overflow = 50 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.401509] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.max_pool_size = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.401671] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.max_retries = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.401839] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.mysql_sql_mode = TRADITIONAL {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.401997] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.mysql_wsrep_sync_wait = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.402169] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.pool_timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.402334] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.retry_interval = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.402512] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.slave_connection = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.402686] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.sqlite_synchronous = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.402849] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] database.use_db_reconnect = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.403039] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.backend = sqlalchemy {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.403213] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.connection = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.403377] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.connection_debug = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.403545] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.connection_parameters = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.403720] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.connection_recycle_time = 3600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.403888] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.connection_trace = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.404062] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.db_inc_retry_interval = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.404230] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.db_max_retries = 20 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.404394] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.db_max_retry_interval = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.404575] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.db_retry_interval = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.404748] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.max_overflow = 50 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.404910] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.max_pool_size = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.405082] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.max_retries = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.405253] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.405413] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.mysql_wsrep_sync_wait = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.405570] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.pool_timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.405731] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.retry_interval = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.405886] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.slave_connection = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.406061] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] api_database.sqlite_synchronous = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.406243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] devices.enabled_mdev_types = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.406420] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.406590] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ephemeral_storage_encryption.default_format = luks {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.406755] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ephemeral_storage_encryption.enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.406917] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ephemeral_storage_encryption.key_size = 512 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.407101] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.api_servers = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.407265] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.407428] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.407591] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.407751] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.connect_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.407909] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.connect_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.408085] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.debug = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.408253] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.default_trusted_certificate_ids = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.408415] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.enable_certificate_validation = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.408578] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.enable_rbd_download = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.408736] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.endpoint_override = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.408902] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.409074] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.409238] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.max_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.409397] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.min_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.409561] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.num_retries = 3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.409731] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.rbd_ceph_conf = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.409894] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.rbd_connect_timeout = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.410072] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.rbd_pool = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.410245] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.rbd_user = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.410406] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.region_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.410567] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.retriable_status_codes = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.410726] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.service_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.410898] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.service_type = image {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.411068] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.411229] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.status_code_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.411387] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.status_code_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.411545] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.411727] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.411891] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.verify_glance_signatures = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.412071] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] glance.version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.412243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] guestfs.debug = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.412425] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] mks.enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.412797] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.412994] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] image_cache.manager_interval = 2400 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.413182] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] image_cache.precache_concurrency = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.413356] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] image_cache.remove_unused_base_images = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.413527] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.413697] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.413873] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] image_cache.subdirectory_name = _base {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.414060] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.api_max_retries = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.414230] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.api_retry_interval = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.414390] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.auth_section = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.414583] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.auth_type = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.414755] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.414916] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.415095] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.415265] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.conductor_group = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.415425] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.connect_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.415589] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.connect_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.415749] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.endpoint_override = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.415911] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.416082] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.416246] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.max_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.416403] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.min_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.416567] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.peer_list = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.416725] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.region_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.416912] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.retriable_status_codes = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.417101] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.serial_console_state_timeout = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.417267] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.service_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.417438] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.service_type = baremetal {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.417598] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.shard = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.417764] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.417923] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.status_code_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.418094] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.status_code_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.418257] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.418438] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.418599] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ironic.version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.418784] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.418958] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] key_manager.fixed_key = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.419159] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.419321] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.barbican_api_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.419481] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.barbican_endpoint = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.419654] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.barbican_endpoint_type = public {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.419815] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.barbican_region_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.419974] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.420148] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.420314] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.420475] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.420632] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.420797] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.number_of_retries = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.420958] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.retry_delay = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.421136] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.send_service_user_token = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.421299] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.421456] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.421616] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.verify_ssl = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.421774] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican.verify_ssl_path = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.421940] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican_service_user.auth_section = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.422115] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican_service_user.auth_type = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.422278] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican_service_user.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.422456] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican_service_user.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.422638] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican_service_user.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.422806] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican_service_user.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.422965] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican_service_user.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.423143] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican_service_user.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.423303] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] barbican_service_user.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.423470] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.approle_role_id = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.423628] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.approle_secret_id = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.423801] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.kv_mountpoint = secret {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.423961] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.kv_path = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.424138] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.kv_version = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.424298] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.namespace = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.424453] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.root_token_id = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.424642] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.ssl_ca_crt_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.424821] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.timeout = 60.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.424987] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.use_ssl = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.425179] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.425351] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.auth_section = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.425516] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.auth_type = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.425676] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.425831] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.425993] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.426167] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.connect_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.426324] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.connect_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.426481] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.endpoint_override = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.426643] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.426804] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.426986] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.max_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.427160] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.min_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.427319] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.region_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.427477] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.retriable_status_codes = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.427637] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.service_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.427805] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.service_type = identity {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.427965] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.428139] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.status_code_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.428299] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.status_code_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.428456] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.428635] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.428798] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] keystone.version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.428996] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.connection_uri = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.429174] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.cpu_mode = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.429340] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.cpu_model_extra_flags = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.429509] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.cpu_models = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.429682] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.cpu_power_governor_high = performance {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.429851] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.cpu_power_governor_low = powersave {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.430023] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.cpu_power_management = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.430199] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.cpu_power_management_strategy = cpu_state {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.430363] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.device_detach_attempts = 8 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.430525] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.device_detach_timeout = 20 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.430693] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.disk_cachemodes = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.430850] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.disk_prefix = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.431022] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.enabled_perf_events = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.431191] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.file_backed_memory = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.431356] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.gid_maps = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.431516] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.hw_disk_discard = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.431675] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.hw_machine_type = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.431843] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.images_rbd_ceph_conf = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.432024] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.432183] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.432352] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.images_rbd_glance_store_name = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.432547] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.images_rbd_pool = rbd {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.432731] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.images_type = default {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.432897] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.images_volume_group = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.433078] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.inject_key = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.433247] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.inject_partition = -2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.433409] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.inject_password = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.433571] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.iscsi_iface = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.433734] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.iser_use_multipath = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.433896] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_bandwidth = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.434070] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_completion_timeout = 800 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.434238] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_downtime = 500 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.434401] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_downtime_delay = 75 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.434594] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_downtime_steps = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.434765] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_inbound_addr = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.434931] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_permit_auto_converge = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.435106] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_permit_post_copy = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.435267] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_scheme = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.435437] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_timeout_action = abort {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.435628] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_tunnelled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.435794] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_uri = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.435963] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.live_migration_with_native_tls = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.436142] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.max_queues = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.436308] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.mem_stats_period_seconds = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.436535] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.migration_inbound_addr = 10.180.1.21 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.436702] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.nfs_mount_options = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.437947] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.nfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.438153] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.num_aoe_discover_tries = 3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.438334] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.num_iser_scan_tries = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.438503] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.num_memory_encrypted_guests = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.438682] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.num_nvme_discover_tries = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.438852] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.num_pcie_ports = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.439033] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.num_volume_scan_tries = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.439210] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.pmem_namespaces = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.439372] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.quobyte_client_cfg = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.439661] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.quobyte_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.439841] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rbd_connect_timeout = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.440019] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.440192] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.440354] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rbd_secret_uuid = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.440512] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rbd_user = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.440685] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.realtime_scheduler_priority = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.440854] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.remote_filesystem_transport = ssh {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.441022] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rescue_image_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.441187] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rescue_kernel_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.441344] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rescue_ramdisk_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.441512] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rng_dev_path = /dev/urandom {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.441672] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.rx_queue_size = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.441838] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.smbfs_mount_options = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.442129] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.smbfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.442305] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.snapshot_compression = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.442491] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.snapshot_image_format = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.442725] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.442896] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.sparse_logical_volumes = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.443075] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.swtpm_enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.443251] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.swtpm_group = tss {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.443417] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.swtpm_user = tss {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.443585] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.sysinfo_serial = unique {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.443748] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.tb_cache_size = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.443907] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.tx_queue_size = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.444086] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.uid_maps = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.444260] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.use_virtio_for_bridges = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.444423] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.virt_type = kvm {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.444616] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.volume_clear = zero {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.444791] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.volume_clear_size = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.444959] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.volume_use_multipath = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.445133] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.vzstorage_cache_path = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.445305] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.445472] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.vzstorage_mount_group = qemu {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.445637] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.vzstorage_mount_opts = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.445806] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.446090] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.446272] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.vzstorage_mount_user = stack {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.446439] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.446611] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.auth_section = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.446791] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.auth_type = password {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.446954] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.447129] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.447296] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.447458] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.connect_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.447619] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.connect_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.447793] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.default_floating_pool = public {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.447954] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.endpoint_override = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.448132] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.extension_sync_interval = 600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.448297] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.http_retries = 3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.448459] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.448620] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.448783] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.max_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.448950] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.metadata_proxy_shared_secret = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.449123] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.min_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.449295] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.ovs_bridge = br-int {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.449460] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.physnets = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.449629] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.region_name = RegionOne {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.449790] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.retriable_status_codes = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.449958] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.service_metadata_proxy = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.450130] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.service_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.450299] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.service_type = network {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.450461] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.450619] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.status_code_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.450780] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.status_code_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.450935] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.451125] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.451753] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] neutron.version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.451753] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] notifications.bdms_in_notifications = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.451753] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] notifications.default_level = INFO {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.451895] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] notifications.notification_format = unversioned {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.451938] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] notifications.notify_on_state_change = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.452124] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.452307] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] pci.alias = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.452499] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] pci.device_spec = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.452677] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] pci.report_in_placement = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.452850] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.auth_section = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.453039] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.auth_type = password {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.453216] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.auth_url = http://10.180.1.21/identity {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.453377] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.453537] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.453703] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.453863] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.connect_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.454039] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.connect_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.454206] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.default_domain_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.454364] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.default_domain_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.454540] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.domain_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.454715] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.domain_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.454876] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.endpoint_override = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.455051] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.455214] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.455371] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.max_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.455530] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.min_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.455702] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.password = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.455861] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.project_domain_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.456040] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.project_domain_name = Default {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.456211] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.project_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.456386] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.project_name = service {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.456554] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.region_name = RegionOne {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.456717] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.retriable_status_codes = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.456875] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.service_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.457055] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.service_type = placement {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.457224] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.457385] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.status_code_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.457547] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.status_code_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.457708] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.system_scope = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.457868] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.458038] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.trust_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.458201] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.user_domain_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.458370] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.user_domain_name = Default {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.458528] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.user_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.458701] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.username = nova {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.458881] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.459053] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] placement.version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.459233] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.cores = 20 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.459396] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.count_usage_from_placement = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.459565] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.459734] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.injected_file_content_bytes = 10240 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.459899] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.injected_file_path_length = 255 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.460075] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.injected_files = 5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.460243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.instances = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.460407] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.key_pairs = 100 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.460571] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.metadata_items = 128 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.460739] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.ram = 51200 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.460900] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.recheck_quota = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.461078] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.server_group_members = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.461246] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] quota.server_groups = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.461423] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.461587] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.461750] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.image_metadata_prefilter = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.461912] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.462087] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.max_attempts = 3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.462253] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.max_placement_results = 1000 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.462428] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.462628] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.query_placement_for_image_type_support = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.462800] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.462973] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] scheduler.workers = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.463160] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.463331] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.463510] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.463682] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.463846] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.464013] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.464186] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.464375] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.enabled_filters = ['ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.464570] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.host_subset_size = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.464797] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.hypervisor_version_weight_multiplier = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.465026] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.image_properties_default_architecture = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.465251] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.465471] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.isolated_hosts = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.465687] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.isolated_images = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.465900] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.max_instances_per_host = 50 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.466160] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.466388] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.num_instances_weight_multiplier = 0.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.466607] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.pci_in_placement = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.466828] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.467060] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.467286] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.467503] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.467722] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.467938] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.468165] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.track_instance_changes = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.468399] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.468623] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] metrics.required = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.468840] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] metrics.weight_multiplier = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.469073] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] metrics.weight_of_unavailable = -10000.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.469299] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] metrics.weight_setting = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.469665] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.469892] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] serial_console.enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.470144] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] serial_console.port_range = 10000:20000 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.470373] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.470597] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.470820] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] serial_console.serialproxy_port = 6083 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.471053] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.auth_section = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.471287] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.auth_type = password {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.471499] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.471709] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.471920] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.472148] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.472363] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.472603] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.send_service_user_token = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.472824] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.473057] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] service_user.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.473287] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.agent_enabled = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.473499] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.473861] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.474120] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.html5proxy_host = 0.0.0.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.474360] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.html5proxy_port = 6082 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.474592] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.image_compression = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.474796] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.jpeg_compression = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.475013] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.playback_compression = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.475247] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.server_listen = 127.0.0.1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.475466] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.475678] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.streaming_mode = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.475886] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] spice.zlib_compression = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.476147] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] upgrade_levels.baseapi = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.476386] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] upgrade_levels.compute = auto {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.476602] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] upgrade_levels.conductor = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.476841] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] upgrade_levels.scheduler = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.477047] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vendordata_dynamic_auth.auth_section = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.477269] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vendordata_dynamic_auth.auth_type = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.477482] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vendordata_dynamic_auth.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.477694] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vendordata_dynamic_auth.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.477914] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vendordata_dynamic_auth.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.478140] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vendordata_dynamic_auth.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.478352] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vendordata_dynamic_auth.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.478567] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vendordata_dynamic_auth.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.478779] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vendordata_dynamic_auth.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.479010] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.api_retry_count = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.479239] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.ca_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.479464] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.cache_prefix = devstack-image-cache {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.479686] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.cluster_name = testcl1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.479904] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.connection_pool_size = 10 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.480131] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.console_delay_seconds = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.480358] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.datastore_regex = ^datastore.* {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.480622] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.host_ip = vc1.osci.c.eu-de-1.cloud.sap {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.480848] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.host_password = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.481086] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.host_port = 443 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.481317] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.host_username = administrator@vsphere.local {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.481546] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.insecure = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.481762] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.integration_bridge = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.481980] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.maximum_objects = 100 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.482206] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.pbm_default_policy = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.482422] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.pbm_enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.482633] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.pbm_wsdl_location = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.482854] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.483087] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.serial_port_proxy_uri = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.483301] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.serial_port_service_uri = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.483521] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.task_poll_interval = 0.5 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.483745] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.use_linked_clone = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.483963] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.vnc_keymap = en-us {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.484193] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.vnc_port = 5900 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.484412] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vmware.vnc_port_total = 10000 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.484674] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.auth_schemes = ['none'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.484907] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.485268] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.485509] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.485733] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.novncproxy_port = 6080 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.485960] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.server_listen = 127.0.0.1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.486232] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.486457] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.vencrypt_ca_certs = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.486672] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.vencrypt_client_cert = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.486894] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vnc.vencrypt_client_key = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.487141] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.487370] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.disable_fallback_pcpu_query = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.487589] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.disable_group_policy_check_upcall = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.487806] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.488035] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.disable_rootwrap = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.488260] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.enable_numa_live_migration = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.488476] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.488692] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.488909] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.handle_virt_lifecycle_events = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.489139] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.libvirt_disable_apic = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.489356] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.never_download_image_if_on_rbd = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.489572] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.qemu_monitor_announce_self_count = 3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.489786] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.qemu_monitor_announce_self_interval = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.490017] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.490243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.skip_cpu_compare_at_startup = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.490458] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.490671] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.490884] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.skip_reserve_in_use_ironic_nodes = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.491112] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.491335] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.491571] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.491791] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.client_socket_timeout = 900 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.492014] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.default_pool_size = 1000 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.492246] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.keep_alive = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.492463] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.max_header_line = 16384 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.492674] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.secure_proxy_ssl_header = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.492892] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.ssl_ca_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.493121] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.ssl_cert_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.493339] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.ssl_key_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.493557] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.tcp_keepidle = 600 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.493791] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.494015] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] zvm.ca_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.494239] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] zvm.cloud_connector_url = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.494595] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] zvm.image_tmp_path = /opt/stack/data/n-cpu-1/images {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.494813] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] zvm.reachable_timeout = 300 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.495064] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.enforce_new_defaults = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.495505] env[63593]: WARNING oslo_config.cfg [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] Deprecated: Option "enforce_scope" from group "oslo_policy" is deprecated for removal (This configuration was added temporarily to facilitate a smooth transition to the new RBAC. OpenStack will always enforce scope checks. This configuration option is deprecated and will be removed in the 2025.2 cycle.). Its value may be silently ignored in the future. [ 549.495730] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.enforce_scope = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.495958] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.policy_default_rule = default {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.496211] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.496468] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.policy_file = policy.yaml {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.496705] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.496923] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.497150] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.497365] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.497578] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.497798] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.498031] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.498269] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.connection_string = messaging:// {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.498493] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.enabled = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.498714] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.es_doc_type = notification {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.498928] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.es_scroll_size = 10000 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.499167] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.es_scroll_time = 2m {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.499387] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.filter_error_trace = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.499608] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.hmac_keys = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.499829] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.sentinel_service_name = mymaster {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.500059] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.socket_timeout = 0.1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.500282] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.trace_requests = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.500495] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler.trace_sqlalchemy = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.500723] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler_jaeger.process_tags = {} {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.500933] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler_jaeger.service_name_prefix = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.501162] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] profiler_otlp.service_name_prefix = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.501386] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] remote_debug.host = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.501597] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] remote_debug.port = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.501827] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.502058] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.502280] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.502495] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.502709] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.502922] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.503153] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.heartbeat_in_pthread = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.503372] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.503592] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.503818] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.hostname = devstack {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.504050] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.504280] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.504508] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.504746] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.504969] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.processname = nova-compute {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.505203] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.505418] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.505646] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.505861] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.506095] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.506317] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_bytes = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.506556] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_length = 0 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.506785] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.507010] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.507240] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.507455] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_stream_fanout = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.507667] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.507880] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rabbit_transient_quorum_queue = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.508114] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.508336] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.ssl = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.508559] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.508782] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.508997] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.509262] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.509482] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.ssl_version = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.509699] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_rabbit.use_queue_manager = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.509936] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.510166] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_notifications.retry = -1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514019] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514019] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_messaging_notifications.transport_url = **** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514019] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.auth_section = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514019] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.auth_type = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514019] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.cafile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514019] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.certfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514019] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.collect_timing = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514265] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.connect_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514265] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.connect_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514265] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.endpoint_id = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514265] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.endpoint_override = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514265] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.insecure = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514265] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.keyfile = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514265] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.max_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514457] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.min_version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514457] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.region_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514457] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.retriable_status_codes = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514457] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.service_name = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514457] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.service_type = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514457] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.split_loggers = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514457] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.status_code_retries = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514671] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.status_code_retry_delay = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514671] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.timeout = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514671] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.valid_interfaces = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.514788] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_limit.version = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.515027] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_reports.file_event_handler = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.515253] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_reports.file_event_handler_interval = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.515466] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] oslo_reports.log_dir = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.515696] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_linux_bridge_privileged.capabilities = [12] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.516129] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_linux_bridge_privileged.group = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.516129] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_linux_bridge_privileged.helper_command = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.516319] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_linux_bridge_privileged.logger_name = oslo_privsep.daemon {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.516555] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_linux_bridge_privileged.thread_pool_size = 8 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.516784] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_linux_bridge_privileged.user = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.517027] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_ovs_privileged.capabilities = [12, 1] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520707] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_ovs_privileged.group = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520707] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_ovs_privileged.helper_command = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520707] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_ovs_privileged.logger_name = oslo_privsep.daemon {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520707] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_ovs_privileged.thread_pool_size = 8 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520707] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] vif_plug_ovs_privileged.user = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520707] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_linux_bridge.flat_interface = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520707] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_linux_bridge.forward_bridge_interface = ['all'] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520939] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_linux_bridge.iptables_bottom_regex = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520939] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_linux_bridge.iptables_drop_action = DROP {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520939] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_linux_bridge.iptables_top_regex = {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520939] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_linux_bridge.network_device_mtu = 1500 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520939] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_linux_bridge.use_ipv6 = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520939] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_linux_bridge.vlan_interface = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.520939] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_ovs.default_qos_type = linux-noop {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_ovs.isolate_vif = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_ovs.network_device_mtu = 1500 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_ovs.ovs_vsctl_timeout = 120 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_ovs.ovsdb_connection = tcp:127.0.0.1:6640 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_ovs.ovsdb_interface = native {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_vif_ovs.per_port_bridge = False {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521243] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_brick.lock_path = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521468] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_brick.wait_mpath_device_attempts = 4 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521468] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] os_brick.wait_mpath_device_interval = 1 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521468] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] privsep_osbrick.capabilities = [21] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521538] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] privsep_osbrick.group = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521719] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] privsep_osbrick.helper_command = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.521957] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] privsep_osbrick.logger_name = os_brick.privileged {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.522486] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] privsep_osbrick.thread_pool_size = 8 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.522486] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] privsep_osbrick.user = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.522703] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] nova_sys_admin.capabilities = [0, 1, 2, 3, 12, 21] {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.523035] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] nova_sys_admin.group = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.523195] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] nova_sys_admin.helper_command = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.523443] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] nova_sys_admin.logger_name = oslo_privsep.daemon {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.523982] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] nova_sys_admin.thread_pool_size = 8 {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.523982] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] nova_sys_admin.user = None {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2826}} [ 549.524112] env[63593]: DEBUG oslo_service.service [None req-ed75e5ae-cf3e-4f6a-87fb-ce7d43eac795 None None] ******************************************************************************** {{(pid=63593) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2830}} [ 549.524794] env[63593]: INFO nova.service [-] Starting compute node (version 0.1.0) [ 550.028706] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Getting list of instances from cluster (obj){ [ 550.028706] env[63593]: value = "domain-c8" [ 550.028706] env[63593]: _type = "ClusterComputeResource" [ 550.028706] env[63593]: } {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 550.029883] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-366403ea-43ca-40c9-951c-c80aff59515d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 550.051506] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Got total of 0 instances {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 550.052103] env[63593]: WARNING nova.virt.vmwareapi.driver [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] The vmwareapi driver is not tested by the OpenStack project nor does it have clear maintainer(s) and thus its quality can not be ensured. It should be considered experimental and may be removed in a future release. If you are using the driver in production please let us know via the openstack-discuss mailing list. [ 550.052612] env[63593]: INFO nova.virt.node [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Generated node identity 7345f17f-3af2-4b0a-9521-0292dc691877 [ 550.052840] env[63593]: INFO nova.virt.node [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Wrote node identity 7345f17f-3af2-4b0a-9521-0292dc691877 to /opt/stack/data/n-cpu-1/compute_id [ 550.558120] env[63593]: WARNING nova.compute.manager [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Compute nodes ['7345f17f-3af2-4b0a-9521-0292dc691877'] for host cpu-1 were not found in the database. If this is the first time this service is starting on this host, then you can ignore this warning. [ 551.564233] env[63593]: INFO nova.compute.manager [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Looking for unclaimed instances stuck in BUILDING status for nodes managed by this host [ 552.572068] env[63593]: WARNING nova.compute.manager [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] No compute node record found for host cpu-1. If this is the first time this service is starting on this host, then you can ignore this warning.: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host cpu-1 could not be found. [ 552.572068] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 552.572068] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 552.572068] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 552.572068] env[63593]: DEBUG nova.compute.resource_tracker [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 552.572068] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd278f8b-a7bf-4ea9-86d2-9a585f67af26 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.580394] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b442c3a1-1d81-4ce5-8c23-725fd293f8f4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.593751] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d56d852-b92d-4bc3-abf2-44ca25e89584 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.599927] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5a08aba-8dc4-45a0-a035-8e7309f2f46e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.629509] env[63593]: DEBUG nova.compute.resource_tracker [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181496MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 552.629842] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 552.630176] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 553.134350] env[63593]: WARNING nova.compute.resource_tracker [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] No compute node record for cpu-1:7345f17f-3af2-4b0a-9521-0292dc691877: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host 7345f17f-3af2-4b0a-9521-0292dc691877 could not be found. [ 553.638095] env[63593]: INFO nova.compute.resource_tracker [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Compute node record created for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 with uuid: 7345f17f-3af2-4b0a-9521-0292dc691877 [ 555.150095] env[63593]: DEBUG nova.compute.resource_tracker [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 555.150095] env[63593]: DEBUG nova.compute.resource_tracker [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 555.314196] env[63593]: INFO nova.scheduler.client.report [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] [req-010221ea-fe61-41a7-b660-401fdac3cc5a] Created resource provider record via placement API for resource provider with UUID 7345f17f-3af2-4b0a-9521-0292dc691877 and name domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28. [ 555.331279] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00a6684c-359e-45f6-baab-e0ef30ee5815 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.338935] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2939e4f-7e4f-43dc-9e0c-4cfd88aa07bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.368133] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a08e4da5-7cd0-4126-a309-5e94e128ab7d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.374731] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f3830ff-1338-45ab-b759-d43953b288f0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.386926] env[63593]: DEBUG nova.compute.provider_tree [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Updating inventory in ProviderTree for provider 7345f17f-3af2-4b0a-9521-0292dc691877 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 555.924159] env[63593]: DEBUG nova.scheduler.client.report [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Updated inventory for provider 7345f17f-3af2-4b0a-9521-0292dc691877 with generation 0 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 555.924401] env[63593]: DEBUG nova.compute.provider_tree [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Updating resource provider 7345f17f-3af2-4b0a-9521-0292dc691877 generation from 0 to 1 during operation: update_inventory {{(pid=63593) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 555.924543] env[63593]: DEBUG nova.compute.provider_tree [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Updating inventory in ProviderTree for provider 7345f17f-3af2-4b0a-9521-0292dc691877 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 555.976075] env[63593]: DEBUG nova.compute.provider_tree [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Updating resource provider 7345f17f-3af2-4b0a-9521-0292dc691877 generation from 1 to 2 during operation: update_traits {{(pid=63593) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 556.480548] env[63593]: DEBUG nova.compute.resource_tracker [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 556.480887] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.851s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.480933] env[63593]: DEBUG nova.service [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Creating RPC server for service compute {{(pid=63593) start /opt/stack/nova/nova/service.py:186}} [ 556.494451] env[63593]: DEBUG nova.service [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] Join ServiceGroup membership for this service compute {{(pid=63593) start /opt/stack/nova/nova/service.py:203}} [ 556.494647] env[63593]: DEBUG nova.servicegroup.drivers.db [None req-6269391a-ec5f-42ba-94c9-3b3ac0ec39d5 None None] DB_Driver: join new ServiceGroup member cpu-1 to the compute group, service = {{(pid=63593) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 590.133191] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "642218a7-18e5-4d6e-bdce-40dfa1efba97" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 590.133514] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "642218a7-18e5-4d6e-bdce-40dfa1efba97" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 590.637774] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 591.181258] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 591.181258] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 591.183149] env[63593]: INFO nova.compute.claims [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 592.243427] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65154b9d-6f53-4cd8-9a23-29a115e7b54d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.252061] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2fdbecc-fdb8-4e18-960b-19885d862c48 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.283890] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7b66ac9-0e40-4a29-958a-071f7b6494ed {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.293496] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f002131d-fbbc-47f4-98b3-519001d22a10 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.311010] env[63593]: DEBUG nova.compute.provider_tree [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 592.814160] env[63593]: DEBUG nova.scheduler.client.report [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 593.317835] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.137s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 593.318423] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 593.825500] env[63593]: DEBUG nova.compute.utils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 593.827484] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 593.828722] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 594.337930] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 594.612958] env[63593]: DEBUG nova.policy [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f66fbf8fe6a645ed84fa3dd55175aa7c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3889b4d282fd4033a9071d2c37a863f7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 595.354021] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 595.385874] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 595.386152] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 595.386327] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 595.386518] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 595.386664] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 595.386971] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 595.387140] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 595.387293] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 595.387701] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 595.387866] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 595.388566] env[63593]: DEBUG nova.virt.hardware [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 595.391069] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-822bb058-c263-43f4-aee8-b96cc37ca397 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.401195] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16771a4a-505f-491c-aa7a-9b26966cc437 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.424473] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27b79862-f191-4897-80fe-0760945a2979 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 595.489784] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Successfully created port: 92e9a8c0-823e-4b5b-9551-4686a01e1c00 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 595.971978] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquiring lock "7044b799-74ed-4711-9b53-08f8a8ddbb78" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 595.972242] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Lock "7044b799-74ed-4711-9b53-08f8a8ddbb78" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.255901] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Acquiring lock "c6cf53d2-56c8-4677-9cf9-d72d75cc922b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.255901] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Lock "c6cf53d2-56c8-4677-9cf9-d72d75cc922b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.477044] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 596.610222] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Acquiring lock "5bde524f-b343-423f-85b4-36cb8a6faa96" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.611398] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Lock "5bde524f-b343-423f-85b4-36cb8a6faa96" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.760096] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 597.017261] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 597.019040] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 597.023303] env[63593]: INFO nova.compute.claims [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 597.113860] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 597.302167] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 597.647286] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 598.150021] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54d79aff-9237-4c5b-bf59-690aae4c8710 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.160468] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2690342-6ae5-4c17-b3ed-e2cee0875fa6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.196076] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bece2095-3c60-414d-aabc-96087c5386a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.203425] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a39ca9a9-f947-49fb-81ae-bf7c132f394d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 598.225368] env[63593]: DEBUG nova.compute.provider_tree [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 598.375501] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "fe213cbb-4bed-487c-9d89-2c6b101b8956" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 598.375738] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "fe213cbb-4bed-487c-9d89-2c6b101b8956" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 598.730346] env[63593]: DEBUG nova.scheduler.client.report [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 598.879225] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 599.059855] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "ed5ced72-49a9-42b5-b112-1ca01c023ae6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.060117] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "ed5ced72-49a9-42b5-b112-1ca01c023ae6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 599.238873] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.220s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 599.240165] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 599.243509] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 1.941s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 599.245311] env[63593]: INFO nova.compute.claims [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 599.418417] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.564629] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 599.626165] env[63593]: ERROR nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00, please check neutron logs for more information. [ 599.626165] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 599.626165] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 599.626165] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 599.626165] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 599.626165] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 599.626165] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 599.626165] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 599.626165] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 599.626165] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 599.626165] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 599.626165] env[63593]: ERROR nova.compute.manager raise self.value [ 599.626165] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 599.626165] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 599.626165] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 599.626165] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 599.626604] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 599.626604] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 599.626604] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00, please check neutron logs for more information. [ 599.626604] env[63593]: ERROR nova.compute.manager [ 599.630263] env[63593]: Traceback (most recent call last): [ 599.630329] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 599.630329] env[63593]: listener.cb(fileno) [ 599.630329] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 599.630329] env[63593]: result = function(*args, **kwargs) [ 599.630329] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 599.630329] env[63593]: return func(*args, **kwargs) [ 599.630329] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 599.630329] env[63593]: raise e [ 599.630329] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 599.630329] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 599.630329] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 599.630329] env[63593]: created_port_ids = self._update_ports_for_instance( [ 599.630329] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 599.630329] env[63593]: with excutils.save_and_reraise_exception(): [ 599.630329] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 599.630329] env[63593]: self.force_reraise() [ 599.630329] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 599.630329] env[63593]: raise self.value [ 599.630329] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 599.630329] env[63593]: updated_port = self._update_port( [ 599.630329] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 599.630329] env[63593]: _ensure_no_port_binding_failure(port) [ 599.630329] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 599.630329] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 599.630329] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00, please check neutron logs for more information. [ 599.630329] env[63593]: Removing descriptor: 14 [ 599.633049] env[63593]: ERROR nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00, please check neutron logs for more information. [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Traceback (most recent call last): [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] yield resources [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self.driver.spawn(context, instance, image_meta, [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self._vmops.spawn(context, instance, image_meta, injected_files, [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] vm_ref = self.build_virtual_machine(instance, [ 599.633049] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] vif_infos = vmwarevif.get_vif_info(self._session, [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] for vif in network_info: [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] return self._sync_wrapper(fn, *args, **kwargs) [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self.wait() [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self[:] = self._gt.wait() [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] return self._exit_event.wait() [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 599.633372] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] result = hub.switch() [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] return self.greenlet.switch() [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] result = function(*args, **kwargs) [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] return func(*args, **kwargs) [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] raise e [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] nwinfo = self.network_api.allocate_for_instance( [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] created_port_ids = self._update_ports_for_instance( [ 599.633679] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] with excutils.save_and_reraise_exception(): [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self.force_reraise() [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] raise self.value [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] updated_port = self._update_port( [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] _ensure_no_port_binding_failure(port) [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] raise exception.PortBindingFailed(port_id=port['id']) [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] nova.exception.PortBindingFailed: Binding failed for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00, please check neutron logs for more information. [ 599.634039] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] [ 599.634355] env[63593]: INFO nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Terminating instance [ 599.635435] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "refresh_cache-642218a7-18e5-4d6e-bdce-40dfa1efba97" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 599.635561] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquired lock "refresh_cache-642218a7-18e5-4d6e-bdce-40dfa1efba97" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 599.635830] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 599.752417] env[63593]: DEBUG nova.compute.utils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 599.756085] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Not allocating networking since 'none' was specified. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 600.105355] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 600.228465] env[63593]: DEBUG nova.compute.manager [req-f6efad51-3429-4226-858e-615e41809cca req-51d36be1-9cfd-4fa8-b0bc-9df5b478694f service nova] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Received event network-changed-92e9a8c0-823e-4b5b-9551-4686a01e1c00 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 600.228465] env[63593]: DEBUG nova.compute.manager [req-f6efad51-3429-4226-858e-615e41809cca req-51d36be1-9cfd-4fa8-b0bc-9df5b478694f service nova] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Refreshing instance network info cache due to event network-changed-92e9a8c0-823e-4b5b-9551-4686a01e1c00. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 600.228465] env[63593]: DEBUG oslo_concurrency.lockutils [req-f6efad51-3429-4226-858e-615e41809cca req-51d36be1-9cfd-4fa8-b0bc-9df5b478694f service nova] Acquiring lock "refresh_cache-642218a7-18e5-4d6e-bdce-40dfa1efba97" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 600.251715] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 600.259192] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 600.446641] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1210e67c-fb14-4d62-82ea-cdff66ab44d6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.463381] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d569c71-3c1b-4cf9-a887-9e1bbe076bd3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.509450] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_power_states {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 600.512542] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6aa2a468-069a-4664-bcf2-cafa0f7fb3fa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.522707] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e90d3ab8-3b2d-4dba-aae1-becb08ae8f72 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.539396] env[63593]: DEBUG nova.compute.provider_tree [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 600.572719] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 600.697622] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 600.697903] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.016959] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Getting list of instances from cluster (obj){ [ 601.016959] env[63593]: value = "domain-c8" [ 601.016959] env[63593]: _type = "ClusterComputeResource" [ 601.016959] env[63593]: } {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 601.018094] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46b273d4-25b1-4ce3-97ec-07c1d1e50362 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.031115] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Got total of 0 instances {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 601.031292] env[63593]: WARNING nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] While synchronizing instance power states, found 3 instances in the database and 0 instances on the hypervisor. [ 601.031491] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 642218a7-18e5-4d6e-bdce-40dfa1efba97 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 601.031686] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 7044b799-74ed-4711-9b53-08f8a8ddbb78 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 601.031838] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid c6cf53d2-56c8-4677-9cf9-d72d75cc922b {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 601.032150] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "642218a7-18e5-4d6e-bdce-40dfa1efba97" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.032546] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "7044b799-74ed-4711-9b53-08f8a8ddbb78" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.032627] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "c6cf53d2-56c8-4677-9cf9-d72d75cc922b" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.032745] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_running_deleted_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 601.033109] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Getting list of instances from cluster (obj){ [ 601.033109] env[63593]: value = "domain-c8" [ 601.033109] env[63593]: _type = "ClusterComputeResource" [ 601.033109] env[63593]: } {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 601.034012] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de1021f5-134e-43a7-aa43-c61a7e728c24 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.050309] env[63593]: DEBUG nova.scheduler.client.report [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 601.056030] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Got total of 0 instances {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 601.079411] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Releasing lock "refresh_cache-642218a7-18e5-4d6e-bdce-40dfa1efba97" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 601.079829] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 601.080021] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 601.080951] env[63593]: DEBUG oslo_concurrency.lockutils [req-f6efad51-3429-4226-858e-615e41809cca req-51d36be1-9cfd-4fa8-b0bc-9df5b478694f service nova] Acquired lock "refresh_cache-642218a7-18e5-4d6e-bdce-40dfa1efba97" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 601.081226] env[63593]: DEBUG nova.network.neutron [req-f6efad51-3429-4226-858e-615e41809cca req-51d36be1-9cfd-4fa8-b0bc-9df5b478694f service nova] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Refreshing network info cache for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 601.082466] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1c5fe8ea-9800-4e53-b3f1-5a9754a4c0a1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.092168] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5897aa8-c79b-448d-94b3-d8cb0e78b712 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.117137] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 642218a7-18e5-4d6e-bdce-40dfa1efba97 could not be found. [ 601.117422] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 601.117839] env[63593]: INFO nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Took 0.04 seconds to destroy the instance on the hypervisor. [ 601.118119] env[63593]: DEBUG oslo.service.loopingcall [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 601.118416] env[63593]: DEBUG nova.compute.manager [-] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 601.118416] env[63593]: DEBUG nova.network.neutron [-] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 601.157994] env[63593]: DEBUG nova.network.neutron [-] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 601.200696] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 601.269425] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 601.300572] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 601.300773] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 601.300928] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 601.301751] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 601.302041] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 601.302093] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 601.302316] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 601.302468] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 601.302634] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 601.302797] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 601.302966] env[63593]: DEBUG nova.virt.hardware [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 601.304168] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43ae6e2b-c9dd-4a7f-8024-c6b236498eb9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.316185] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8bce69a-4eee-4173-b868-0c805448b7b9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.332527] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Instance VIF info [] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 601.342198] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Creating folder: OpenStack. Parent ref: group-v4. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 601.342673] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-fc88268f-0248-4c1f-9f64-c462746b9f1d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.354339] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Created folder: OpenStack in parent group-v4. [ 601.354420] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Creating folder: Project (808759a2557b4952ba801e73cb6dd753). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 601.354647] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f5a79d33-34a8-4e63-abd3-4df4b1f4a351 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.364187] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Created folder: Project (808759a2557b4952ba801e73cb6dd753) in parent group-v291016. [ 601.364377] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Creating folder: Instances. Parent ref: group-v291017. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 601.364607] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-630ac507-c46d-4abd-babb-1c709338e522 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.372416] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Created folder: Instances in parent group-v291017. [ 601.373096] env[63593]: DEBUG oslo.service.loopingcall [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 601.373096] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 601.373096] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-d2eb145c-dee9-4158-b88b-6facf12f71cc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.390452] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 601.390452] env[63593]: value = "task-1367928" [ 601.390452] env[63593]: _type = "Task" [ 601.390452] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 601.400357] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367928, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 601.557738] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.314s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 601.558338] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 601.561498] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.915s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.563832] env[63593]: INFO nova.compute.claims [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 601.660851] env[63593]: DEBUG nova.network.neutron [-] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 601.665255] env[63593]: DEBUG nova.network.neutron [req-f6efad51-3429-4226-858e-615e41809cca req-51d36be1-9cfd-4fa8-b0bc-9df5b478694f service nova] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 601.732832] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.782780] env[63593]: DEBUG nova.network.neutron [req-f6efad51-3429-4226-858e-615e41809cca req-51d36be1-9cfd-4fa8-b0bc-9df5b478694f service nova] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 601.913188] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "70709096-265f-4531-8946-51029ebbb998" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.913436] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "70709096-265f-4531-8946-51029ebbb998" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.915888] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367928, 'name': CreateVM_Task, 'duration_secs': 0.384511} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 601.916331] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 601.921019] env[63593]: DEBUG oslo_vmware.service [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d8b216b-b791-49fe-8875-1992265ab93e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.928594] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 601.929694] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 601.930514] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 601.931751] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a76a7415-5c2b-43a7-ab91-0555c2690ffa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.938900] env[63593]: DEBUG oslo_vmware.api [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Waiting for the task: (returnval){ [ 601.938900] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52510317-600a-4819-612e-df1c5ca5e5e3" [ 601.938900] env[63593]: _type = "Task" [ 601.938900] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 601.948977] env[63593]: DEBUG oslo_vmware.api [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52510317-600a-4819-612e-df1c5ca5e5e3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 602.068717] env[63593]: DEBUG nova.compute.utils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 602.072792] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 602.073090] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 602.140817] env[63593]: DEBUG nova.policy [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ce6694d81e5d46ba8eb60d09d59d8e54', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e2d84214996948bd884880e929ab9449', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 602.164662] env[63593]: INFO nova.compute.manager [-] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Took 1.05 seconds to deallocate network for instance. [ 602.170925] env[63593]: DEBUG nova.compute.claims [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 602.171108] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 602.287945] env[63593]: DEBUG oslo_concurrency.lockutils [req-f6efad51-3429-4226-858e-615e41809cca req-51d36be1-9cfd-4fa8-b0bc-9df5b478694f service nova] Releasing lock "refresh_cache-642218a7-18e5-4d6e-bdce-40dfa1efba97" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 602.419041] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 602.453870] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 602.454171] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 602.454407] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 602.454546] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 602.455213] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 602.455491] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-54186050-8fa1-4b0f-b1f1-12895aa64145 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.474465] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 602.474734] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 602.475933] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24b29da8-506d-4be1-af54-836367688b2f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.489468] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-caf079bf-5f59-47fc-aea0-736048cd5215 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.492466] env[63593]: DEBUG oslo_vmware.api [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Waiting for the task: (returnval){ [ 602.492466] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5221655f-c716-a61f-c256-ade2a187e3dc" [ 602.492466] env[63593]: _type = "Task" [ 602.492466] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 602.506022] env[63593]: DEBUG oslo_vmware.api [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5221655f-c716-a61f-c256-ade2a187e3dc, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 602.587532] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 602.741452] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d72ea862-47f2-44e2-b1ce-7c57e841a7eb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.749514] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5620e53f-dcf0-4d9d-858c-47ef476bfd29 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.784366] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f30a2b5-8f19-4d58-881c-34a6626a688d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.792762] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cad25cec-19f1-48b6-9aa2-0b132f35604a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.807381] env[63593]: DEBUG nova.compute.provider_tree [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 602.950088] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 603.004266] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 603.010427] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Creating directory with path [datastore2] vmware_temp/7ecb1281-97e2-44cc-afcf-db640b2c903d/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 603.011813] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-2637904b-9490-4ed1-b52b-de40a3945c80 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.054339] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Created directory with path [datastore2] vmware_temp/7ecb1281-97e2-44cc-afcf-db640b2c903d/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 603.054541] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Fetch image to [datastore2] vmware_temp/7ecb1281-97e2-44cc-afcf-db640b2c903d/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 603.055800] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/7ecb1281-97e2-44cc-afcf-db640b2c903d/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 603.055800] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24c4ffe4-759b-46d8-9da0-6ce27ff8995c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.064685] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec6c815a-56bb-4a58-ab32-e05a93b3bf43 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.076663] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5022c62-cbe4-4cec-8287-3da09538999e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.122234] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-598d61ca-0198-4b64-af74-8420e12cda46 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.128954] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-8e698551-346b-4541-9eaf-a4d7635c7c96 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.222239] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 603.263364] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Successfully created port: 5c70f15e-da1c-4068-affa-90fa094410b3 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 603.314287] env[63593]: DEBUG nova.scheduler.client.report [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 603.325067] env[63593]: DEBUG oslo_vmware.rw_handles [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7ecb1281-97e2-44cc-afcf-db640b2c903d/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 603.410021] env[63593]: DEBUG nova.compute.manager [req-c8d35b53-efd1-4612-a821-ceb6f6efaefb req-182194f9-4769-4b9b-8abd-13623f3956d3 service nova] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Received event network-vif-deleted-92e9a8c0-823e-4b5b-9551-4686a01e1c00 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 603.411415] env[63593]: DEBUG oslo_vmware.rw_handles [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 603.411603] env[63593]: DEBUG oslo_vmware.rw_handles [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7ecb1281-97e2-44cc-afcf-db640b2c903d/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 603.627826] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 603.662928] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 603.663164] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 603.663312] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 603.663655] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 603.663655] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 603.663895] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 603.663946] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 603.664128] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 603.664265] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 603.664424] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 603.666463] env[63593]: DEBUG nova.virt.hardware [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 603.666463] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba095416-1767-459c-931d-f56d5d737fc3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.675490] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71f2e5eb-dda8-421d-a279-77e49689a45d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.833264] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.271s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 603.833531] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 603.836670] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 4.418s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.840725] env[63593]: INFO nova.compute.claims [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 604.342472] env[63593]: DEBUG nova.compute.utils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 604.354126] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 604.354342] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 604.562175] env[63593]: DEBUG nova.policy [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '52d842d5655c4cc98c40e16aa463be11', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c3e4474109984ce6a84e47a5ea02f0e4', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 604.859331] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 605.035383] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 605.035854] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 605.036153] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 605.036335] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 605.049019] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94f33d26-e99d-4f4b-8021-9ba56b8fb5a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.061854] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5bcbc269-5fc1-4220-b9f2-c5cab8534b78 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.105466] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d833063-4172-44cc-93e4-dfc5920d9b27 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.112787] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25802766-0731-49f2-9619-868238f21a06 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.127068] env[63593]: DEBUG nova.compute.provider_tree [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 605.546019] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 605.546019] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 605.546019] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 605.546019] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 605.546019] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 605.546019] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 605.546619] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 605.546619] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 605.546619] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 605.546619] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 605.546619] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 605.546619] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 605.546805] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 605.547041] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 605.632127] env[63593]: DEBUG nova.scheduler.client.report [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 605.873168] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 605.910629] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 605.910886] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 605.910886] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 605.911664] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 605.911904] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 605.912101] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 605.912327] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 605.912487] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 605.912658] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 605.913057] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 605.913057] env[63593]: DEBUG nova.virt.hardware [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 605.913918] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e72f410a-ca99-4765-a67e-c3c3aadc3117 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.925788] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-538b0aae-e834-40bd-a3db-018ef9855abd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.053730] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 606.138883] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.302s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 606.139451] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 606.143933] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.040s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 606.145306] env[63593]: INFO nova.compute.claims [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 606.491041] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Successfully created port: 8e62cb6b-450f-4267-9f34-0b055d9af4ea {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 606.645696] env[63593]: DEBUG nova.compute.utils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 606.649029] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 606.649029] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 606.895428] env[63593]: DEBUG nova.policy [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '33be5e80c65444eaaf2e78b4ee5e497a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cfe14210e5b1405d85ade6e6d022b215', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 607.123512] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquiring lock "65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 607.123778] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Lock "65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 607.151472] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 607.415723] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8215b46b-554b-42ff-bd7a-453f3e55cd63 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.426106] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7be0b547-95e7-4d31-9b18-84b14995eb17 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.463492] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4638deff-55e8-42eb-b2fb-ae7fc7017b0a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.471166] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c7f3162-e88e-4d49-9c7e-4ebcea6302a5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.485710] env[63593]: DEBUG nova.compute.provider_tree [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 607.628661] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 607.989707] env[63593]: DEBUG nova.scheduler.client.report [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 608.162982] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 608.162982] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 608.203203] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 608.203599] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 608.205734] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 608.205958] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 608.207153] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 608.207153] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 608.208508] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 608.208594] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 608.208713] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 608.209360] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 608.209360] env[63593]: DEBUG nova.virt.hardware [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 608.210352] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cace2b4-0643-48b9-838c-e4ea637e53d6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.223934] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e9ffd77-e3b5-4d5b-8677-3acbfe4b1916 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.497578] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.354s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 608.500353] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 608.502537] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.771s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 608.504032] env[63593]: INFO nova.compute.claims [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 608.956026] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Successfully created port: 39592e6f-19be-44dd-8888-4b060eb6f166 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 609.005974] env[63593]: DEBUG nova.compute.utils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 609.007755] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 609.007959] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 609.123115] env[63593]: DEBUG nova.policy [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '11f69140e45743e1bd026f9a339d8657', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9349407b79b3418a81536c91cb87c815', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 609.513666] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 609.812885] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1313160-59d4-4478-970c-bd29390a8858 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.823203] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1922a166-cfac-4469-998e-b901e38d7be8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.863070] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c06e9fd4-eb7d-40a2-86a8-878262c97cd5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.872091] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67d4d183-b75b-4e23-a1d1-91ca39a527db {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.886193] env[63593]: DEBUG nova.compute.provider_tree [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 610.118166] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Acquiring lock "d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 610.118471] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Lock "d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 610.390130] env[63593]: DEBUG nova.scheduler.client.report [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 610.512236] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Successfully created port: 56519d14-008a-4cfe-a540-50b419d4570d {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 610.537612] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 610.563509] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 610.565177] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 610.565177] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 610.565177] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 610.565177] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 610.565177] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 610.565399] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 610.565399] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 610.565399] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 610.565399] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 610.568055] env[63593]: DEBUG nova.virt.hardware [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 610.568507] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dacbf015-814a-4efe-9d80-d5add848b72a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.579169] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d121c42-b5b2-4f91-b32e-f33ebba2caee {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.622335] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 610.629245] env[63593]: ERROR nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 5c70f15e-da1c-4068-affa-90fa094410b3, please check neutron logs for more information. [ 610.629245] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 610.629245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 610.629245] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 610.629245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 610.629245] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 610.629245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 610.629245] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 610.629245] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 610.629245] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 610.629245] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 610.629245] env[63593]: ERROR nova.compute.manager raise self.value [ 610.629245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 610.629245] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 610.629245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 610.629245] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 610.629798] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 610.629798] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 610.629798] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 5c70f15e-da1c-4068-affa-90fa094410b3, please check neutron logs for more information. [ 610.629798] env[63593]: ERROR nova.compute.manager [ 610.629798] env[63593]: Traceback (most recent call last): [ 610.629798] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 610.629798] env[63593]: listener.cb(fileno) [ 610.629798] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 610.629798] env[63593]: result = function(*args, **kwargs) [ 610.629798] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 610.629798] env[63593]: return func(*args, **kwargs) [ 610.629798] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 610.629798] env[63593]: raise e [ 610.629798] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 610.629798] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 610.629798] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 610.629798] env[63593]: created_port_ids = self._update_ports_for_instance( [ 610.629798] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 610.629798] env[63593]: with excutils.save_and_reraise_exception(): [ 610.629798] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 610.629798] env[63593]: self.force_reraise() [ 610.629798] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 610.629798] env[63593]: raise self.value [ 610.629798] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 610.629798] env[63593]: updated_port = self._update_port( [ 610.629798] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 610.629798] env[63593]: _ensure_no_port_binding_failure(port) [ 610.629798] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 610.629798] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 610.630632] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 5c70f15e-da1c-4068-affa-90fa094410b3, please check neutron logs for more information. [ 610.630632] env[63593]: Removing descriptor: 14 [ 610.630632] env[63593]: ERROR nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 5c70f15e-da1c-4068-affa-90fa094410b3, please check neutron logs for more information. [ 610.630632] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Traceback (most recent call last): [ 610.630632] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 610.630632] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] yield resources [ 610.630632] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 610.630632] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self.driver.spawn(context, instance, image_meta, [ 610.630632] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 610.630632] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 610.630632] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 610.630632] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] vm_ref = self.build_virtual_machine(instance, [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] vif_infos = vmwarevif.get_vif_info(self._session, [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] for vif in network_info: [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] return self._sync_wrapper(fn, *args, **kwargs) [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self.wait() [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self[:] = self._gt.wait() [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] return self._exit_event.wait() [ 610.630932] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] result = hub.switch() [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] return self.greenlet.switch() [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] result = function(*args, **kwargs) [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] return func(*args, **kwargs) [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] raise e [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] nwinfo = self.network_api.allocate_for_instance( [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 610.631254] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] created_port_ids = self._update_ports_for_instance( [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] with excutils.save_and_reraise_exception(): [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self.force_reraise() [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] raise self.value [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] updated_port = self._update_port( [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] _ensure_no_port_binding_failure(port) [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 610.631562] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] raise exception.PortBindingFailed(port_id=port['id']) [ 610.632186] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] nova.exception.PortBindingFailed: Binding failed for port 5c70f15e-da1c-4068-affa-90fa094410b3, please check neutron logs for more information. [ 610.632186] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] [ 610.632186] env[63593]: INFO nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Terminating instance [ 610.634821] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Acquiring lock "refresh_cache-c6cf53d2-56c8-4677-9cf9-d72d75cc922b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 610.635058] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Acquired lock "refresh_cache-c6cf53d2-56c8-4677-9cf9-d72d75cc922b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 610.635237] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 610.901027] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.396s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 610.901027] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 610.902063] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.731s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 611.141984] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 611.203790] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 611.410249] env[63593]: DEBUG nova.compute.utils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 611.419511] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 611.419805] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 611.547289] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Acquiring lock "43bcc8c6-0fd5-467b-9e11-3b788cf04236" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 611.548210] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Lock "43bcc8c6-0fd5-467b-9e11-3b788cf04236" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 611.576647] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Acquiring lock "0dbcfda1-a081-4299-a117-3a1974e2510c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 611.577618] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Lock "0dbcfda1-a081-4299-a117-3a1974e2510c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 611.620937] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d65d6be-5d25-457a-af41-aa501dd0e905 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.628811] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7af2ab99-16ed-4979-be20-a29ff09d49d9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.665861] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db4b72b5-5d20-40ff-b030-41add25fb8a9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.674958] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9095015-c273-4676-8f61-51e420022c9f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.691532] env[63593]: DEBUG nova.compute.provider_tree [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 611.745575] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 611.751890] env[63593]: ERROR nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea, please check neutron logs for more information. [ 611.751890] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 611.751890] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 611.751890] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 611.751890] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 611.751890] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 611.751890] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 611.751890] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 611.751890] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 611.751890] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 611.751890] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 611.751890] env[63593]: ERROR nova.compute.manager raise self.value [ 611.751890] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 611.751890] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 611.751890] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 611.751890] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 611.752410] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 611.752410] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 611.752410] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea, please check neutron logs for more information. [ 611.752410] env[63593]: ERROR nova.compute.manager [ 611.752410] env[63593]: Traceback (most recent call last): [ 611.752410] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 611.752410] env[63593]: listener.cb(fileno) [ 611.752410] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 611.752410] env[63593]: result = function(*args, **kwargs) [ 611.752410] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 611.752410] env[63593]: return func(*args, **kwargs) [ 611.752410] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 611.752410] env[63593]: raise e [ 611.752410] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 611.752410] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 611.752410] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 611.752410] env[63593]: created_port_ids = self._update_ports_for_instance( [ 611.752410] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 611.752410] env[63593]: with excutils.save_and_reraise_exception(): [ 611.752410] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 611.752410] env[63593]: self.force_reraise() [ 611.752410] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 611.752410] env[63593]: raise self.value [ 611.752410] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 611.752410] env[63593]: updated_port = self._update_port( [ 611.752410] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 611.752410] env[63593]: _ensure_no_port_binding_failure(port) [ 611.752410] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 611.752410] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 611.753113] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea, please check neutron logs for more information. [ 611.753113] env[63593]: Removing descriptor: 17 [ 611.753748] env[63593]: ERROR nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea, please check neutron logs for more information. [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Traceback (most recent call last): [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] yield resources [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self.driver.spawn(context, instance, image_meta, [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self._vmops.spawn(context, instance, image_meta, injected_files, [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] vm_ref = self.build_virtual_machine(instance, [ 611.753748] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] vif_infos = vmwarevif.get_vif_info(self._session, [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] for vif in network_info: [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] return self._sync_wrapper(fn, *args, **kwargs) [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self.wait() [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self[:] = self._gt.wait() [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] return self._exit_event.wait() [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 611.754341] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] result = hub.switch() [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] return self.greenlet.switch() [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] result = function(*args, **kwargs) [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] return func(*args, **kwargs) [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] raise e [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] nwinfo = self.network_api.allocate_for_instance( [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] created_port_ids = self._update_ports_for_instance( [ 611.754873] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] with excutils.save_and_reraise_exception(): [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self.force_reraise() [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] raise self.value [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] updated_port = self._update_port( [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] _ensure_no_port_binding_failure(port) [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] raise exception.PortBindingFailed(port_id=port['id']) [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] nova.exception.PortBindingFailed: Binding failed for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea, please check neutron logs for more information. [ 611.755484] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] [ 611.755983] env[63593]: INFO nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Terminating instance [ 611.755983] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Acquiring lock "refresh_cache-5bde524f-b343-423f-85b4-36cb8a6faa96" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 611.755983] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Acquired lock "refresh_cache-5bde524f-b343-423f-85b4-36cb8a6faa96" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 611.755983] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 611.798274] env[63593]: DEBUG nova.policy [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '11f69140e45743e1bd026f9a339d8657', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9349407b79b3418a81536c91cb87c815', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 611.920757] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 612.196125] env[63593]: DEBUG nova.scheduler.client.report [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 612.248785] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Releasing lock "refresh_cache-c6cf53d2-56c8-4677-9cf9-d72d75cc922b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 612.249511] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 612.249511] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 612.250295] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-adb42613-828e-4c8b-bcac-905561de3a8f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.268891] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f88d7cb1-0cd5-4b4b-abbe-c577a2760118 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.298116] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c6cf53d2-56c8-4677-9cf9-d72d75cc922b could not be found. [ 612.298116] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 612.298207] env[63593]: INFO nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Took 0.05 seconds to destroy the instance on the hypervisor. [ 612.299445] env[63593]: DEBUG oslo.service.loopingcall [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 612.299445] env[63593]: DEBUG nova.compute.manager [-] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 612.299445] env[63593]: DEBUG nova.network.neutron [-] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 612.344167] env[63593]: DEBUG nova.network.neutron [-] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 612.379231] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 612.721693] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.819s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 612.727971] env[63593]: ERROR nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00, please check neutron logs for more information. [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Traceback (most recent call last): [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self.driver.spawn(context, instance, image_meta, [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self._vmops.spawn(context, instance, image_meta, injected_files, [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] vm_ref = self.build_virtual_machine(instance, [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] vif_infos = vmwarevif.get_vif_info(self._session, [ 612.727971] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] for vif in network_info: [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] return self._sync_wrapper(fn, *args, **kwargs) [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self.wait() [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self[:] = self._gt.wait() [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] return self._exit_event.wait() [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] result = hub.switch() [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 612.728554] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] return self.greenlet.switch() [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] result = function(*args, **kwargs) [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] return func(*args, **kwargs) [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] raise e [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] nwinfo = self.network_api.allocate_for_instance( [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] created_port_ids = self._update_ports_for_instance( [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] with excutils.save_and_reraise_exception(): [ 612.728880] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] self.force_reraise() [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] raise self.value [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] updated_port = self._update_port( [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] _ensure_no_port_binding_failure(port) [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] raise exception.PortBindingFailed(port_id=port['id']) [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] nova.exception.PortBindingFailed: Binding failed for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00, please check neutron logs for more information. [ 612.729208] env[63593]: ERROR nova.compute.manager [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] [ 612.729498] env[63593]: DEBUG nova.compute.utils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Binding failed for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 612.729498] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.774s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 612.729498] env[63593]: INFO nova.compute.claims [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 612.739577] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Build of instance 642218a7-18e5-4d6e-bdce-40dfa1efba97 was re-scheduled: Binding failed for port 92e9a8c0-823e-4b5b-9551-4686a01e1c00, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 612.740332] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 612.740645] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "refresh_cache-642218a7-18e5-4d6e-bdce-40dfa1efba97" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 612.740843] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquired lock "refresh_cache-642218a7-18e5-4d6e-bdce-40dfa1efba97" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 612.741074] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 612.850271] env[63593]: DEBUG nova.network.neutron [-] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 612.892255] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "81b0b694-b9fb-4416-8751-7f1e24e96869" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 612.892656] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "81b0b694-b9fb-4416-8751-7f1e24e96869" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 612.940065] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 612.949170] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 612.982748] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 612.986301] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 612.986301] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 612.986301] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 612.986301] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 612.986301] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 612.986786] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 612.986786] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 612.986786] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 612.986786] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 612.986786] env[63593]: DEBUG nova.virt.hardware [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 612.987092] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f1a11c6-67ed-40dd-93a4-b024038895c6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.001227] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3062a17a-1326-48b9-88fb-e3ad31d06d34 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.057381] env[63593]: DEBUG nova.compute.manager [req-80b51d21-6a9b-4a65-909a-266db076ba79 req-b59ac9f8-a339-4a33-a6bf-d1f132da2e7a service nova] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Received event network-changed-5c70f15e-da1c-4068-affa-90fa094410b3 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 613.057381] env[63593]: DEBUG nova.compute.manager [req-80b51d21-6a9b-4a65-909a-266db076ba79 req-b59ac9f8-a339-4a33-a6bf-d1f132da2e7a service nova] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Refreshing instance network info cache due to event network-changed-5c70f15e-da1c-4068-affa-90fa094410b3. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 613.057381] env[63593]: DEBUG oslo_concurrency.lockutils [req-80b51d21-6a9b-4a65-909a-266db076ba79 req-b59ac9f8-a339-4a33-a6bf-d1f132da2e7a service nova] Acquiring lock "refresh_cache-c6cf53d2-56c8-4677-9cf9-d72d75cc922b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 613.057381] env[63593]: DEBUG oslo_concurrency.lockutils [req-80b51d21-6a9b-4a65-909a-266db076ba79 req-b59ac9f8-a339-4a33-a6bf-d1f132da2e7a service nova] Acquired lock "refresh_cache-c6cf53d2-56c8-4677-9cf9-d72d75cc922b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 613.057381] env[63593]: DEBUG nova.network.neutron [req-80b51d21-6a9b-4a65-909a-266db076ba79 req-b59ac9f8-a339-4a33-a6bf-d1f132da2e7a service nova] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Refreshing network info cache for port 5c70f15e-da1c-4068-affa-90fa094410b3 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 613.284050] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Successfully created port: 27bc5beb-cb5f-470e-91ff-918686f9a252 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 613.299220] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 613.358121] env[63593]: INFO nova.compute.manager [-] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Took 1.06 seconds to deallocate network for instance. [ 613.363839] env[63593]: DEBUG nova.compute.claims [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 613.363839] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.455733] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Releasing lock "refresh_cache-5bde524f-b343-423f-85b4-36cb8a6faa96" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 613.456209] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 613.456392] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 613.456755] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c669fc14-a060-4faf-a958-74b615415fe1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.468380] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4adf4136-7ee8-4354-ae6f-caab987f930e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.496512] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5bde524f-b343-423f-85b4-36cb8a6faa96 could not be found. [ 613.496512] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 613.496512] env[63593]: INFO nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Took 0.04 seconds to destroy the instance on the hypervisor. [ 613.496512] env[63593]: DEBUG oslo.service.loopingcall [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 613.496512] env[63593]: DEBUG nova.compute.manager [-] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 613.496512] env[63593]: DEBUG nova.network.neutron [-] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 613.515263] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 613.572430] env[63593]: DEBUG nova.network.neutron [-] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 613.579393] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Acquiring lock "c6d88cbe-488b-490b-a37c-51a311276dfe" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.579619] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Lock "c6d88cbe-488b-490b-a37c-51a311276dfe" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.620310] env[63593]: DEBUG nova.network.neutron [req-80b51d21-6a9b-4a65-909a-266db076ba79 req-b59ac9f8-a339-4a33-a6bf-d1f132da2e7a service nova] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 613.994258] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d222017e-92b5-4352-9966-bddc4285e5dd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 613.999646] env[63593]: DEBUG nova.compute.manager [req-1131d098-0e33-4621-ad67-03f20a7de915 req-ecea760d-15f1-45c0-9332-f009b15b9a97 service nova] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Received event network-changed-8e62cb6b-450f-4267-9f34-0b055d9af4ea {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 613.999646] env[63593]: DEBUG nova.compute.manager [req-1131d098-0e33-4621-ad67-03f20a7de915 req-ecea760d-15f1-45c0-9332-f009b15b9a97 service nova] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Refreshing instance network info cache due to event network-changed-8e62cb6b-450f-4267-9f34-0b055d9af4ea. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 613.999767] env[63593]: DEBUG oslo_concurrency.lockutils [req-1131d098-0e33-4621-ad67-03f20a7de915 req-ecea760d-15f1-45c0-9332-f009b15b9a97 service nova] Acquiring lock "refresh_cache-5bde524f-b343-423f-85b4-36cb8a6faa96" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 614.000094] env[63593]: DEBUG oslo_concurrency.lockutils [req-1131d098-0e33-4621-ad67-03f20a7de915 req-ecea760d-15f1-45c0-9332-f009b15b9a97 service nova] Acquired lock "refresh_cache-5bde524f-b343-423f-85b4-36cb8a6faa96" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 614.000348] env[63593]: DEBUG nova.network.neutron [req-1131d098-0e33-4621-ad67-03f20a7de915 req-ecea760d-15f1-45c0-9332-f009b15b9a97 service nova] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Refreshing network info cache for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 614.002095] env[63593]: DEBUG nova.network.neutron [req-80b51d21-6a9b-4a65-909a-266db076ba79 req-b59ac9f8-a339-4a33-a6bf-d1f132da2e7a service nova] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 614.014426] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-935f4563-d3f0-4a11-a3a6-d06915ad5c00 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 614.020783] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Releasing lock "refresh_cache-642218a7-18e5-4d6e-bdce-40dfa1efba97" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 614.021040] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 614.021213] env[63593]: DEBUG nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 614.021345] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 614.052685] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58099b69-0b24-442d-9558-7cdb4680c38f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 614.060609] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70dfe7f0-e3c1-4097-b875-4f37fb37bcda {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 614.076576] env[63593]: DEBUG nova.compute.provider_tree [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 614.081455] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 614.081455] env[63593]: DEBUG nova.network.neutron [-] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 614.509368] env[63593]: DEBUG oslo_concurrency.lockutils [req-80b51d21-6a9b-4a65-909a-266db076ba79 req-b59ac9f8-a339-4a33-a6bf-d1f132da2e7a service nova] Releasing lock "refresh_cache-c6cf53d2-56c8-4677-9cf9-d72d75cc922b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 614.544190] env[63593]: ERROR nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 39592e6f-19be-44dd-8888-4b060eb6f166, please check neutron logs for more information. [ 614.544190] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 614.544190] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 614.544190] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 614.544190] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 614.544190] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 614.544190] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 614.544190] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 614.544190] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 614.544190] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 614.544190] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 614.544190] env[63593]: ERROR nova.compute.manager raise self.value [ 614.544190] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 614.544190] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 614.544190] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 614.544190] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 614.544662] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 614.544662] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 614.544662] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 39592e6f-19be-44dd-8888-4b060eb6f166, please check neutron logs for more information. [ 614.544662] env[63593]: ERROR nova.compute.manager [ 614.544662] env[63593]: Traceback (most recent call last): [ 614.544662] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 614.544662] env[63593]: listener.cb(fileno) [ 614.544662] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 614.544662] env[63593]: result = function(*args, **kwargs) [ 614.544662] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 614.544662] env[63593]: return func(*args, **kwargs) [ 614.544662] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 614.544662] env[63593]: raise e [ 614.544662] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 614.544662] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 614.544662] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 614.544662] env[63593]: created_port_ids = self._update_ports_for_instance( [ 614.544662] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 614.544662] env[63593]: with excutils.save_and_reraise_exception(): [ 614.544662] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 614.544662] env[63593]: self.force_reraise() [ 614.544662] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 614.544662] env[63593]: raise self.value [ 614.544662] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 614.544662] env[63593]: updated_port = self._update_port( [ 614.544662] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 614.544662] env[63593]: _ensure_no_port_binding_failure(port) [ 614.544662] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 614.544662] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 614.545525] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 39592e6f-19be-44dd-8888-4b060eb6f166, please check neutron logs for more information. [ 614.545525] env[63593]: Removing descriptor: 18 [ 614.545525] env[63593]: ERROR nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 39592e6f-19be-44dd-8888-4b060eb6f166, please check neutron logs for more information. [ 614.545525] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Traceback (most recent call last): [ 614.545525] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 614.545525] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] yield resources [ 614.545525] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 614.545525] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self.driver.spawn(context, instance, image_meta, [ 614.545525] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 614.545525] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self._vmops.spawn(context, instance, image_meta, injected_files, [ 614.545525] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 614.545525] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] vm_ref = self.build_virtual_machine(instance, [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] vif_infos = vmwarevif.get_vif_info(self._session, [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] for vif in network_info: [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] return self._sync_wrapper(fn, *args, **kwargs) [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self.wait() [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self[:] = self._gt.wait() [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] return self._exit_event.wait() [ 614.545915] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] result = hub.switch() [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] return self.greenlet.switch() [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] result = function(*args, **kwargs) [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] return func(*args, **kwargs) [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] raise e [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] nwinfo = self.network_api.allocate_for_instance( [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 614.546301] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] created_port_ids = self._update_ports_for_instance( [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] with excutils.save_and_reraise_exception(): [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self.force_reraise() [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] raise self.value [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] updated_port = self._update_port( [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] _ensure_no_port_binding_failure(port) [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 614.546643] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] raise exception.PortBindingFailed(port_id=port['id']) [ 614.546966] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] nova.exception.PortBindingFailed: Binding failed for port 39592e6f-19be-44dd-8888-4b060eb6f166, please check neutron logs for more information. [ 614.546966] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] [ 614.546966] env[63593]: INFO nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Terminating instance [ 614.549149] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "refresh_cache-fe213cbb-4bed-487c-9d89-2c6b101b8956" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 614.549149] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquired lock "refresh_cache-fe213cbb-4bed-487c-9d89-2c6b101b8956" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 614.549149] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 614.551326] env[63593]: DEBUG nova.network.neutron [req-1131d098-0e33-4621-ad67-03f20a7de915 req-ecea760d-15f1-45c0-9332-f009b15b9a97 service nova] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 614.584250] env[63593]: DEBUG nova.scheduler.client.report [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 614.586160] env[63593]: DEBUG nova.network.neutron [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 614.587033] env[63593]: INFO nova.compute.manager [-] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Took 1.09 seconds to deallocate network for instance. [ 614.596696] env[63593]: DEBUG nova.compute.claims [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 614.596696] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 614.703147] env[63593]: DEBUG nova.network.neutron [req-1131d098-0e33-4621-ad67-03f20a7de915 req-ecea760d-15f1-45c0-9332-f009b15b9a97 service nova] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 614.868607] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Acquiring lock "6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 614.869151] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Lock "6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 615.090652] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.366s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 615.091750] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 615.099567] env[63593]: INFO nova.compute.manager [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] Took 1.08 seconds to deallocate network for instance. [ 615.102054] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 9.048s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 615.102240] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 615.102396] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 615.102675] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.942s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 615.108118] env[63593]: INFO nova.compute.claims [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 615.113071] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2eae825b-8a59-41f2-a034-604d317d6e7a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.123944] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 615.129407] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f61ccd5e-de98-4a53-8b11-a29ea3da59c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.147629] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca74d18e-3466-4ed1-85a6-75e0471d1516 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.157833] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-703db6bc-eb0f-4d1e-b4e6-8897c32abe98 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 615.195472] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181487MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 615.195714] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 615.205822] env[63593]: DEBUG oslo_concurrency.lockutils [req-1131d098-0e33-4621-ad67-03f20a7de915 req-ecea760d-15f1-45c0-9332-f009b15b9a97 service nova] Releasing lock "refresh_cache-5bde524f-b343-423f-85b4-36cb8a6faa96" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 615.601414] env[63593]: DEBUG nova.compute.utils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 615.604897] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 615.604897] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 615.619340] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 615.791101] env[63593]: DEBUG nova.policy [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '11f69140e45743e1bd026f9a339d8657', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9349407b79b3418a81536c91cb87c815', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 616.104557] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 616.129025] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Releasing lock "refresh_cache-fe213cbb-4bed-487c-9d89-2c6b101b8956" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 616.129025] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 616.129025] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 616.129025] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d66e1c91-115a-4245-a2f0-db48cbc0d38f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.147485] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f60a534c-df84-4a88-b659-b33dbbaf2f5e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.167840] env[63593]: INFO nova.scheduler.client.report [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Deleted allocations for instance 642218a7-18e5-4d6e-bdce-40dfa1efba97 [ 616.184514] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fe213cbb-4bed-487c-9d89-2c6b101b8956 could not be found. [ 616.184889] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 616.185190] env[63593]: INFO nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Took 0.06 seconds to destroy the instance on the hypervisor. [ 616.185559] env[63593]: DEBUG oslo.service.loopingcall [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 616.186793] env[63593]: DEBUG nova.compute.manager [-] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 616.187017] env[63593]: DEBUG nova.network.neutron [-] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 616.297065] env[63593]: DEBUG nova.network.neutron [-] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 616.478955] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1d53d47-a42f-49ab-9ad8-1e4498060f47 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.492134] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de08c323-5a00-4e33-befd-58ac71c71bfb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.532131] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a6bffc3-d4ba-47f0-ae89-6a381e74a53b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.541191] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf48e7a0-3f7c-4799-a134-fa35d6f0c169 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.564234] env[63593]: DEBUG nova.compute.provider_tree [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 616.585448] env[63593]: DEBUG nova.compute.manager [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Received event network-vif-deleted-5c70f15e-da1c-4068-affa-90fa094410b3 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 616.585658] env[63593]: DEBUG nova.compute.manager [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Received event network-changed-39592e6f-19be-44dd-8888-4b060eb6f166 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 616.591149] env[63593]: DEBUG nova.compute.manager [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Refreshing instance network info cache due to event network-changed-39592e6f-19be-44dd-8888-4b060eb6f166. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 616.591149] env[63593]: DEBUG oslo_concurrency.lockutils [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] Acquiring lock "refresh_cache-fe213cbb-4bed-487c-9d89-2c6b101b8956" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 616.591149] env[63593]: DEBUG oslo_concurrency.lockutils [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] Acquired lock "refresh_cache-fe213cbb-4bed-487c-9d89-2c6b101b8956" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 616.591149] env[63593]: DEBUG nova.network.neutron [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Refreshing network info cache for port 39592e6f-19be-44dd-8888-4b060eb6f166 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 616.689188] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ecb13b76-9265-4f9c-8408-2ac8f9e482a4 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "642218a7-18e5-4d6e-bdce-40dfa1efba97" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 26.556s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 616.690426] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "642218a7-18e5-4d6e-bdce-40dfa1efba97" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 15.658s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 616.690763] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 642218a7-18e5-4d6e-bdce-40dfa1efba97] During sync_power_state the instance has a pending task (spawning). Skip. [ 616.690854] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "642218a7-18e5-4d6e-bdce-40dfa1efba97" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 616.800396] env[63593]: DEBUG nova.network.neutron [-] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 617.068195] env[63593]: DEBUG nova.scheduler.client.report [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 617.082527] env[63593]: ERROR nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 56519d14-008a-4cfe-a540-50b419d4570d, please check neutron logs for more information. [ 617.082527] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 617.082527] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 617.082527] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 617.082527] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 617.082527] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 617.082527] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 617.082527] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 617.082527] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 617.082527] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 617.082527] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 617.082527] env[63593]: ERROR nova.compute.manager raise self.value [ 617.082527] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 617.082527] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 617.082527] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 617.082527] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 617.082941] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 617.082941] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 617.082941] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 56519d14-008a-4cfe-a540-50b419d4570d, please check neutron logs for more information. [ 617.082941] env[63593]: ERROR nova.compute.manager [ 617.082941] env[63593]: Traceback (most recent call last): [ 617.082941] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 617.082941] env[63593]: listener.cb(fileno) [ 617.082941] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 617.082941] env[63593]: result = function(*args, **kwargs) [ 617.082941] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 617.082941] env[63593]: return func(*args, **kwargs) [ 617.082941] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 617.082941] env[63593]: raise e [ 617.082941] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 617.082941] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 617.082941] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 617.082941] env[63593]: created_port_ids = self._update_ports_for_instance( [ 617.082941] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 617.082941] env[63593]: with excutils.save_and_reraise_exception(): [ 617.082941] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 617.082941] env[63593]: self.force_reraise() [ 617.082941] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 617.082941] env[63593]: raise self.value [ 617.082941] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 617.082941] env[63593]: updated_port = self._update_port( [ 617.082941] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 617.082941] env[63593]: _ensure_no_port_binding_failure(port) [ 617.082941] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 617.082941] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 617.083673] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 56519d14-008a-4cfe-a540-50b419d4570d, please check neutron logs for more information. [ 617.083673] env[63593]: Removing descriptor: 19 [ 617.083673] env[63593]: ERROR nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 56519d14-008a-4cfe-a540-50b419d4570d, please check neutron logs for more information. [ 617.083673] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Traceback (most recent call last): [ 617.083673] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 617.083673] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] yield resources [ 617.083673] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 617.083673] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self.driver.spawn(context, instance, image_meta, [ 617.083673] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 617.083673] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 617.083673] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 617.083673] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] vm_ref = self.build_virtual_machine(instance, [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] vif_infos = vmwarevif.get_vif_info(self._session, [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] for vif in network_info: [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] return self._sync_wrapper(fn, *args, **kwargs) [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self.wait() [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self[:] = self._gt.wait() [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] return self._exit_event.wait() [ 617.083958] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] result = hub.switch() [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] return self.greenlet.switch() [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] result = function(*args, **kwargs) [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] return func(*args, **kwargs) [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] raise e [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] nwinfo = self.network_api.allocate_for_instance( [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 617.084293] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] created_port_ids = self._update_ports_for_instance( [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] with excutils.save_and_reraise_exception(): [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self.force_reraise() [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] raise self.value [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] updated_port = self._update_port( [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] _ensure_no_port_binding_failure(port) [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 617.084616] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] raise exception.PortBindingFailed(port_id=port['id']) [ 617.084904] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] nova.exception.PortBindingFailed: Binding failed for port 56519d14-008a-4cfe-a540-50b419d4570d, please check neutron logs for more information. [ 617.084904] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] [ 617.084904] env[63593]: INFO nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Terminating instance [ 617.085973] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "refresh_cache-ed5ced72-49a9-42b5-b112-1ca01c023ae6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 617.086146] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquired lock "refresh_cache-ed5ced72-49a9-42b5-b112-1ca01c023ae6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 617.086335] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 617.116585] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 617.141139] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:20Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='84',id=12,is_public=True,memory_mb=256,name='m1.micro',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 617.141615] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 617.141615] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 617.141734] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 617.141912] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 617.142314] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 617.142766] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 617.142766] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 617.142861] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 617.142943] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 617.143131] env[63593]: DEBUG nova.virt.hardware [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 617.144217] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cef3f899-96e9-4726-aab0-67edda93f97f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 617.147325] env[63593]: DEBUG nova.network.neutron [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 617.154668] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-500f889f-934d-42bc-a624-a77e4d7ef652 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 617.192817] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 617.306352] env[63593]: INFO nova.compute.manager [-] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Took 1.12 seconds to deallocate network for instance. [ 617.308816] env[63593]: DEBUG nova.compute.claims [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 617.308908] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.434036] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Successfully created port: 3ccc6572-840d-4160-95ba-8f2564a96711 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 617.551159] env[63593]: DEBUG nova.network.neutron [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 617.575150] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.472s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 617.575702] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 617.578220] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.436s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 617.579629] env[63593]: INFO nova.compute.claims [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 617.632966] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 617.651961] env[63593]: DEBUG nova.compute.manager [req-6210bc86-48bf-464b-b062-e8131ed8c719 req-9fbf372b-bd0e-49ae-be97-725545759faf service nova] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Received event network-vif-deleted-8e62cb6b-450f-4267-9f34-0b055d9af4ea {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 617.652177] env[63593]: DEBUG nova.compute.manager [req-6210bc86-48bf-464b-b062-e8131ed8c719 req-9fbf372b-bd0e-49ae-be97-725545759faf service nova] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Received event network-changed-56519d14-008a-4cfe-a540-50b419d4570d {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 617.652343] env[63593]: DEBUG nova.compute.manager [req-6210bc86-48bf-464b-b062-e8131ed8c719 req-9fbf372b-bd0e-49ae-be97-725545759faf service nova] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Refreshing instance network info cache due to event network-changed-56519d14-008a-4cfe-a540-50b419d4570d. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 617.652525] env[63593]: DEBUG oslo_concurrency.lockutils [req-6210bc86-48bf-464b-b062-e8131ed8c719 req-9fbf372b-bd0e-49ae-be97-725545759faf service nova] Acquiring lock "refresh_cache-ed5ced72-49a9-42b5-b112-1ca01c023ae6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 617.723875] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.921914] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 618.053680] env[63593]: DEBUG oslo_concurrency.lockutils [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] Releasing lock "refresh_cache-fe213cbb-4bed-487c-9d89-2c6b101b8956" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 618.054781] env[63593]: DEBUG nova.compute.manager [req-e00c8c85-4f3b-4f36-9cf9-6f7b80841d34 req-e9fb1090-1fc1-4a77-b1fd-49b20000a07b service nova] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Received event network-vif-deleted-39592e6f-19be-44dd-8888-4b060eb6f166 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 618.060420] env[63593]: ERROR nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 27bc5beb-cb5f-470e-91ff-918686f9a252, please check neutron logs for more information. [ 618.060420] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 618.060420] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 618.060420] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 618.060420] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 618.060420] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 618.060420] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 618.060420] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 618.060420] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 618.060420] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 618.060420] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 618.060420] env[63593]: ERROR nova.compute.manager raise self.value [ 618.060420] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 618.060420] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 618.060420] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 618.060420] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 618.060857] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 618.060857] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 618.060857] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 27bc5beb-cb5f-470e-91ff-918686f9a252, please check neutron logs for more information. [ 618.060857] env[63593]: ERROR nova.compute.manager [ 618.061894] env[63593]: Traceback (most recent call last): [ 618.062103] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 618.062103] env[63593]: listener.cb(fileno) [ 618.062103] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 618.062103] env[63593]: result = function(*args, **kwargs) [ 618.062103] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 618.062103] env[63593]: return func(*args, **kwargs) [ 618.062103] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 618.062103] env[63593]: raise e [ 618.062320] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 618.062320] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 618.062320] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 618.062320] env[63593]: created_port_ids = self._update_ports_for_instance( [ 618.062320] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 618.062320] env[63593]: with excutils.save_and_reraise_exception(): [ 618.062320] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 618.062320] env[63593]: self.force_reraise() [ 618.062320] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 618.062320] env[63593]: raise self.value [ 618.062320] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 618.062320] env[63593]: updated_port = self._update_port( [ 618.062320] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 618.062320] env[63593]: _ensure_no_port_binding_failure(port) [ 618.062320] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 618.062320] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 618.062320] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 27bc5beb-cb5f-470e-91ff-918686f9a252, please check neutron logs for more information. [ 618.062320] env[63593]: Removing descriptor: 21 [ 618.063097] env[63593]: ERROR nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 27bc5beb-cb5f-470e-91ff-918686f9a252, please check neutron logs for more information. [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Traceback (most recent call last): [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] yield resources [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self.driver.spawn(context, instance, image_meta, [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self._vmops.spawn(context, instance, image_meta, injected_files, [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] vm_ref = self.build_virtual_machine(instance, [ 618.063097] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] vif_infos = vmwarevif.get_vif_info(self._session, [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] for vif in network_info: [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] return self._sync_wrapper(fn, *args, **kwargs) [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self.wait() [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self[:] = self._gt.wait() [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] return self._exit_event.wait() [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 618.063399] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] result = hub.switch() [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] return self.greenlet.switch() [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] result = function(*args, **kwargs) [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] return func(*args, **kwargs) [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] raise e [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] nwinfo = self.network_api.allocate_for_instance( [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] created_port_ids = self._update_ports_for_instance( [ 618.063779] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] with excutils.save_and_reraise_exception(): [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self.force_reraise() [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] raise self.value [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] updated_port = self._update_port( [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] _ensure_no_port_binding_failure(port) [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] raise exception.PortBindingFailed(port_id=port['id']) [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] nova.exception.PortBindingFailed: Binding failed for port 27bc5beb-cb5f-470e-91ff-918686f9a252, please check neutron logs for more information. [ 618.064163] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] [ 618.064544] env[63593]: INFO nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Terminating instance [ 618.068663] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "refresh_cache-fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 618.068663] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquired lock "refresh_cache-fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 618.068663] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 618.089194] env[63593]: DEBUG nova.compute.utils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 618.091705] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Not allocating networking since 'none' was specified. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 618.430537] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Releasing lock "refresh_cache-ed5ced72-49a9-42b5-b112-1ca01c023ae6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 618.431100] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 618.431100] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 618.431368] env[63593]: DEBUG oslo_concurrency.lockutils [req-6210bc86-48bf-464b-b062-e8131ed8c719 req-9fbf372b-bd0e-49ae-be97-725545759faf service nova] Acquired lock "refresh_cache-ed5ced72-49a9-42b5-b112-1ca01c023ae6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 618.431552] env[63593]: DEBUG nova.network.neutron [req-6210bc86-48bf-464b-b062-e8131ed8c719 req-9fbf372b-bd0e-49ae-be97-725545759faf service nova] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Refreshing network info cache for port 56519d14-008a-4cfe-a540-50b419d4570d {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 618.433513] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-12da8b8c-a786-432f-a5a4-df2332ec7d10 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.447572] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb90ff61-9f8c-4196-8d81-06827c75cfd2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.474985] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ed5ced72-49a9-42b5-b112-1ca01c023ae6 could not be found. [ 618.475066] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 618.475614] env[63593]: INFO nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Took 0.04 seconds to destroy the instance on the hypervisor. [ 618.476112] env[63593]: DEBUG oslo.service.loopingcall [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 618.476710] env[63593]: DEBUG nova.compute.manager [-] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 618.476710] env[63593]: DEBUG nova.network.neutron [-] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 618.555722] env[63593]: DEBUG nova.network.neutron [-] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 618.595561] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 618.647836] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 618.838391] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-224ac349-606e-4d2a-9954-a0ecb91acd81 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.847639] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2cc633d-8966-43cc-ae95-9d8b624b4382 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.884365] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e886c66c-09ee-431c-ae53-e6020e529693 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.893956] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93ba9933-4412-48cc-bdd8-636a27007c2b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.908496] env[63593]: DEBUG nova.compute.provider_tree [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 618.965781] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 618.977045] env[63593]: DEBUG nova.network.neutron [req-6210bc86-48bf-464b-b062-e8131ed8c719 req-9fbf372b-bd0e-49ae-be97-725545759faf service nova] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 619.059271] env[63593]: DEBUG nova.network.neutron [-] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 619.214299] env[63593]: DEBUG nova.network.neutron [req-6210bc86-48bf-464b-b062-e8131ed8c719 req-9fbf372b-bd0e-49ae-be97-725545759faf service nova] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 619.411753] env[63593]: DEBUG nova.scheduler.client.report [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 619.468536] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Releasing lock "refresh_cache-fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 619.468960] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 619.469155] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 619.469435] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8aac7af5-9b25-47fe-9e9c-ec96eb5c7dd4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 619.482855] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a452e92-3b5e-4e47-a8b0-b891bba36c73 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 619.513745] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fa1bd569-3db7-4b96-b0b2-d6acf6ad6567 could not be found. [ 619.513962] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 619.514988] env[63593]: INFO nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Took 0.05 seconds to destroy the instance on the hypervisor. [ 619.514988] env[63593]: DEBUG oslo.service.loopingcall [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 619.514988] env[63593]: DEBUG nova.compute.manager [-] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 619.514988] env[63593]: DEBUG nova.network.neutron [-] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 619.562877] env[63593]: INFO nova.compute.manager [-] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Took 1.09 seconds to deallocate network for instance. [ 619.563880] env[63593]: DEBUG nova.network.neutron [-] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 619.570145] env[63593]: DEBUG nova.compute.claims [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 619.570338] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 619.605606] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 619.641686] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 619.642852] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 619.642852] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 619.642852] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 619.642852] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 619.642852] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 619.643096] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 619.643997] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 619.644732] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 619.645022] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 619.645231] env[63593]: DEBUG nova.virt.hardware [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 619.646489] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-358b293d-33d8-41f5-926f-a1297922cf29 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 619.658367] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88ac49c0-1e29-4890-a2a9-eefd26bfde27 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 619.673420] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Instance VIF info [] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 619.679227] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Creating folder: Project (9af34d1cbeec41ef9e6a44afff00384b). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 619.680456] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-5102882e-f81b-4656-8854-5d9c00689e98 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 619.690271] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Created folder: Project (9af34d1cbeec41ef9e6a44afff00384b) in parent group-v291016. [ 619.690557] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Creating folder: Instances. Parent ref: group-v291020. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 619.690701] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0533b3a0-20da-456d-a9a8-156ae470293c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 619.700628] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Created folder: Instances in parent group-v291020. [ 619.700858] env[63593]: DEBUG oslo.service.loopingcall [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 619.701259] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 619.701259] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-9f25cecb-2301-49c3-b0e7-400a45d630c6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 619.721712] env[63593]: DEBUG oslo_concurrency.lockutils [req-6210bc86-48bf-464b-b062-e8131ed8c719 req-9fbf372b-bd0e-49ae-be97-725545759faf service nova] Releasing lock "refresh_cache-ed5ced72-49a9-42b5-b112-1ca01c023ae6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 619.722082] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 619.722082] env[63593]: value = "task-1367931" [ 619.722082] env[63593]: _type = "Task" [ 619.722082] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 619.730881] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367931, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 619.926122] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.345s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 619.926122] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 619.926937] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 6.564s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 620.071249] env[63593]: DEBUG nova.network.neutron [-] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 620.239335] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367931, 'name': CreateVM_Task, 'duration_secs': 0.302848} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 620.239514] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 620.239908] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 620.240145] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 620.240416] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 620.241031] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-629f946c-69c3-4636-befb-a49ebebeb0f8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.247316] env[63593]: DEBUG oslo_vmware.api [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Waiting for the task: (returnval){ [ 620.247316] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52bf88aa-9c58-161f-19fe-bd429ebc075c" [ 620.247316] env[63593]: _type = "Task" [ 620.247316] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 620.256548] env[63593]: DEBUG oslo_vmware.api [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52bf88aa-9c58-161f-19fe-bd429ebc075c, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 620.432870] env[63593]: DEBUG nova.compute.utils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 620.434130] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 620.434130] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 620.578947] env[63593]: INFO nova.compute.manager [-] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Took 1.06 seconds to deallocate network for instance. [ 620.581252] env[63593]: DEBUG nova.policy [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c7dbfd823a3b4b26b06a2edd82d4bc48', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '1c19c9d9be6842dcae43079f6756a31b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 620.586333] env[63593]: DEBUG nova.compute.claims [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 620.587090] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 620.601861] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "bb3a0a79-ef74-4fb0-a286-91b1b92df751" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 620.602094] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "bb3a0a79-ef74-4fb0-a286-91b1b92df751" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 620.733994] env[63593]: DEBUG nova.compute.manager [req-ec8b90d7-3a3b-43d4-9fbf-c3ac49c839c2 req-49364f0c-8028-4828-9ebe-9259e3af6673 service nova] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Received event network-changed-27bc5beb-cb5f-470e-91ff-918686f9a252 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 620.735571] env[63593]: DEBUG nova.compute.manager [req-ec8b90d7-3a3b-43d4-9fbf-c3ac49c839c2 req-49364f0c-8028-4828-9ebe-9259e3af6673 service nova] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Refreshing instance network info cache due to event network-changed-27bc5beb-cb5f-470e-91ff-918686f9a252. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 620.735781] env[63593]: DEBUG oslo_concurrency.lockutils [req-ec8b90d7-3a3b-43d4-9fbf-c3ac49c839c2 req-49364f0c-8028-4828-9ebe-9259e3af6673 service nova] Acquiring lock "refresh_cache-fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 620.735998] env[63593]: DEBUG oslo_concurrency.lockutils [req-ec8b90d7-3a3b-43d4-9fbf-c3ac49c839c2 req-49364f0c-8028-4828-9ebe-9259e3af6673 service nova] Acquired lock "refresh_cache-fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 620.736104] env[63593]: DEBUG nova.network.neutron [req-ec8b90d7-3a3b-43d4-9fbf-c3ac49c839c2 req-49364f0c-8028-4828-9ebe-9259e3af6673 service nova] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Refreshing network info cache for port 27bc5beb-cb5f-470e-91ff-918686f9a252 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 620.757932] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 620.758374] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 620.758722] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 620.767256] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aee020bb-d143-488c-b76c-1b35211f8aca {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.775292] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f41ec57d-bf03-4f2c-be74-e9da9859f4b4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.811907] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21e23e9f-7e99-4a82-a7e3-230db0256d8d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.820255] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bbfbe15-bc48-45e5-a724-8271be01e61e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.838719] env[63593]: DEBUG nova.compute.provider_tree [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 620.937833] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 621.026167] env[63593]: ERROR nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3ccc6572-840d-4160-95ba-8f2564a96711, please check neutron logs for more information. [ 621.026167] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 621.026167] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 621.026167] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 621.026167] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 621.026167] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 621.026167] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 621.026167] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 621.026167] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 621.026167] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 621.026167] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 621.026167] env[63593]: ERROR nova.compute.manager raise self.value [ 621.026167] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 621.026167] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 621.026167] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 621.026167] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 621.026868] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 621.026868] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 621.026868] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3ccc6572-840d-4160-95ba-8f2564a96711, please check neutron logs for more information. [ 621.026868] env[63593]: ERROR nova.compute.manager [ 621.026868] env[63593]: Traceback (most recent call last): [ 621.026868] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 621.026868] env[63593]: listener.cb(fileno) [ 621.026868] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 621.026868] env[63593]: result = function(*args, **kwargs) [ 621.026868] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 621.026868] env[63593]: return func(*args, **kwargs) [ 621.026868] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 621.026868] env[63593]: raise e [ 621.026868] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 621.026868] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 621.026868] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 621.026868] env[63593]: created_port_ids = self._update_ports_for_instance( [ 621.026868] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 621.026868] env[63593]: with excutils.save_and_reraise_exception(): [ 621.026868] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 621.026868] env[63593]: self.force_reraise() [ 621.026868] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 621.026868] env[63593]: raise self.value [ 621.026868] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 621.026868] env[63593]: updated_port = self._update_port( [ 621.026868] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 621.026868] env[63593]: _ensure_no_port_binding_failure(port) [ 621.026868] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 621.026868] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 621.027924] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 3ccc6572-840d-4160-95ba-8f2564a96711, please check neutron logs for more information. [ 621.027924] env[63593]: Removing descriptor: 18 [ 621.027924] env[63593]: ERROR nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3ccc6572-840d-4160-95ba-8f2564a96711, please check neutron logs for more information. [ 621.027924] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] Traceback (most recent call last): [ 621.027924] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 621.027924] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] yield resources [ 621.027924] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 621.027924] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self.driver.spawn(context, instance, image_meta, [ 621.027924] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 621.027924] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self._vmops.spawn(context, instance, image_meta, injected_files, [ 621.027924] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 621.027924] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] vm_ref = self.build_virtual_machine(instance, [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] vif_infos = vmwarevif.get_vif_info(self._session, [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] for vif in network_info: [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] return self._sync_wrapper(fn, *args, **kwargs) [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self.wait() [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self[:] = self._gt.wait() [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] return self._exit_event.wait() [ 621.028479] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] result = hub.switch() [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] return self.greenlet.switch() [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] result = function(*args, **kwargs) [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] return func(*args, **kwargs) [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] raise e [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] nwinfo = self.network_api.allocate_for_instance( [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 621.028993] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] created_port_ids = self._update_ports_for_instance( [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] with excutils.save_and_reraise_exception(): [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self.force_reraise() [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] raise self.value [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] updated_port = self._update_port( [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] _ensure_no_port_binding_failure(port) [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 621.029867] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] raise exception.PortBindingFailed(port_id=port['id']) [ 621.030403] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] nova.exception.PortBindingFailed: Binding failed for port 3ccc6572-840d-4160-95ba-8f2564a96711, please check neutron logs for more information. [ 621.030403] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] [ 621.030403] env[63593]: INFO nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Terminating instance [ 621.031720] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "refresh_cache-70709096-265f-4531-8946-51029ebbb998" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 621.031873] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquired lock "refresh_cache-70709096-265f-4531-8946-51029ebbb998" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 621.032045] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 621.081866] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Successfully created port: c5d5c11c-4a35-4e6c-bfcd-444efec65695 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 621.288780] env[63593]: DEBUG nova.network.neutron [req-ec8b90d7-3a3b-43d4-9fbf-c3ac49c839c2 req-49364f0c-8028-4828-9ebe-9259e3af6673 service nova] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 621.342091] env[63593]: DEBUG nova.scheduler.client.report [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 621.577275] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 621.587096] env[63593]: DEBUG nova.network.neutron [req-ec8b90d7-3a3b-43d4-9fbf-c3ac49c839c2 req-49364f0c-8028-4828-9ebe-9259e3af6673 service nova] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 621.847251] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.920s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 621.848195] env[63593]: ERROR nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 5c70f15e-da1c-4068-affa-90fa094410b3, please check neutron logs for more information. [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Traceback (most recent call last): [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self.driver.spawn(context, instance, image_meta, [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] vm_ref = self.build_virtual_machine(instance, [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] vif_infos = vmwarevif.get_vif_info(self._session, [ 621.848195] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] for vif in network_info: [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] return self._sync_wrapper(fn, *args, **kwargs) [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self.wait() [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self[:] = self._gt.wait() [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] return self._exit_event.wait() [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] result = hub.switch() [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 621.848554] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] return self.greenlet.switch() [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] result = function(*args, **kwargs) [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] return func(*args, **kwargs) [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] raise e [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] nwinfo = self.network_api.allocate_for_instance( [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] created_port_ids = self._update_ports_for_instance( [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] with excutils.save_and_reraise_exception(): [ 621.848898] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] self.force_reraise() [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] raise self.value [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] updated_port = self._update_port( [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] _ensure_no_port_binding_failure(port) [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] raise exception.PortBindingFailed(port_id=port['id']) [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] nova.exception.PortBindingFailed: Binding failed for port 5c70f15e-da1c-4068-affa-90fa094410b3, please check neutron logs for more information. [ 621.849243] env[63593]: ERROR nova.compute.manager [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] [ 621.849696] env[63593]: DEBUG nova.compute.utils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Binding failed for port 5c70f15e-da1c-4068-affa-90fa094410b3, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 621.850338] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 7.256s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 621.856985] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Build of instance c6cf53d2-56c8-4677-9cf9-d72d75cc922b was re-scheduled: Binding failed for port 5c70f15e-da1c-4068-affa-90fa094410b3, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 621.856985] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 621.856985] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Acquiring lock "refresh_cache-c6cf53d2-56c8-4677-9cf9-d72d75cc922b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 621.856985] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Acquired lock "refresh_cache-c6cf53d2-56c8-4677-9cf9-d72d75cc922b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 621.857236] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 621.902145] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 621.922908] env[63593]: DEBUG nova.compute.manager [req-af5e6044-9003-4ff1-8df5-8849f4895868 req-f9eaf66d-ffea-4c10-bfb1-13ea278c97b8 service nova] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Received event network-vif-deleted-56519d14-008a-4cfe-a540-50b419d4570d {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 621.951909] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 621.990123] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 621.990123] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 621.990123] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 621.990292] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 621.990292] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 621.990292] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 621.990292] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 621.990292] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 621.990437] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 621.990437] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 621.990437] env[63593]: DEBUG nova.virt.hardware [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 621.990956] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1c209fa-1dee-4c9d-bb13-3da4a5c498c3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.006986] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74bc5c2a-250e-4027-8c39-ac98ff865792 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.090318] env[63593]: DEBUG oslo_concurrency.lockutils [req-ec8b90d7-3a3b-43d4-9fbf-c3ac49c839c2 req-49364f0c-8028-4828-9ebe-9259e3af6673 service nova] Releasing lock "refresh_cache-fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 622.091484] env[63593]: DEBUG nova.compute.manager [req-ec8b90d7-3a3b-43d4-9fbf-c3ac49c839c2 req-49364f0c-8028-4828-9ebe-9259e3af6673 service nova] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Received event network-vif-deleted-27bc5beb-cb5f-470e-91ff-918686f9a252 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 622.335261] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Acquiring lock "4869e3e4-48bb-446f-a15e-3a53bae1bc9b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.335261] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Lock "4869e3e4-48bb-446f-a15e-3a53bae1bc9b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.404461] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 622.406491] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Releasing lock "refresh_cache-70709096-265f-4531-8946-51029ebbb998" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 622.406867] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 622.407361] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 622.407690] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8fcd0a47-b1e4-49b7-8393-7b522d9a33c0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.420724] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ba9b85c-989e-4612-82b0-0988d658a5d1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.446960] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 70709096-265f-4531-8946-51029ebbb998 could not be found. [ 622.447223] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 622.447413] env[63593]: INFO nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Took 0.04 seconds to destroy the instance on the hypervisor. [ 622.447713] env[63593]: DEBUG oslo.service.loopingcall [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 622.447880] env[63593]: DEBUG nova.compute.manager [-] [instance: 70709096-265f-4531-8946-51029ebbb998] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 622.447995] env[63593]: DEBUG nova.network.neutron [-] [instance: 70709096-265f-4531-8946-51029ebbb998] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 622.496696] env[63593]: DEBUG nova.network.neutron [-] [instance: 70709096-265f-4531-8946-51029ebbb998] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 622.732598] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91285d33-efb7-4864-a5d9-d3929b888888 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.742130] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82d13cc6-2cd8-4a21-b707-7ef0fa4a57c4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.747289] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 622.780211] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d14d1466-b6fd-4e26-b466-c87a7d0e61b0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.788030] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7951e12-19aa-4065-8da2-dac6988f84ee {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.801891] env[63593]: DEBUG nova.compute.provider_tree [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 622.998741] env[63593]: DEBUG nova.network.neutron [-] [instance: 70709096-265f-4531-8946-51029ebbb998] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 623.250047] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Releasing lock "refresh_cache-c6cf53d2-56c8-4677-9cf9-d72d75cc922b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 623.250127] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 623.250284] env[63593]: DEBUG nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 623.250521] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 623.287891] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 623.307117] env[63593]: DEBUG nova.scheduler.client.report [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 623.357456] env[63593]: DEBUG nova.compute.manager [req-9abcb5e7-d3a0-438e-8d7e-3a19d96b74d6 req-220d6375-a72c-445c-801b-70798870cfee service nova] [instance: 70709096-265f-4531-8946-51029ebbb998] Received event network-changed-3ccc6572-840d-4160-95ba-8f2564a96711 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 623.357555] env[63593]: DEBUG nova.compute.manager [req-9abcb5e7-d3a0-438e-8d7e-3a19d96b74d6 req-220d6375-a72c-445c-801b-70798870cfee service nova] [instance: 70709096-265f-4531-8946-51029ebbb998] Refreshing instance network info cache due to event network-changed-3ccc6572-840d-4160-95ba-8f2564a96711. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 623.358074] env[63593]: DEBUG oslo_concurrency.lockutils [req-9abcb5e7-d3a0-438e-8d7e-3a19d96b74d6 req-220d6375-a72c-445c-801b-70798870cfee service nova] Acquiring lock "refresh_cache-70709096-265f-4531-8946-51029ebbb998" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 623.358074] env[63593]: DEBUG oslo_concurrency.lockutils [req-9abcb5e7-d3a0-438e-8d7e-3a19d96b74d6 req-220d6375-a72c-445c-801b-70798870cfee service nova] Acquired lock "refresh_cache-70709096-265f-4531-8946-51029ebbb998" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 623.358528] env[63593]: DEBUG nova.network.neutron [req-9abcb5e7-d3a0-438e-8d7e-3a19d96b74d6 req-220d6375-a72c-445c-801b-70798870cfee service nova] [instance: 70709096-265f-4531-8946-51029ebbb998] Refreshing network info cache for port 3ccc6572-840d-4160-95ba-8f2564a96711 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 623.506018] env[63593]: INFO nova.compute.manager [-] [instance: 70709096-265f-4531-8946-51029ebbb998] Took 1.06 seconds to deallocate network for instance. [ 623.511878] env[63593]: DEBUG nova.compute.claims [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 623.511878] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 623.628108] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "d46033b4-b3d5-494c-b25c-106475fe4409" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 623.628385] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "d46033b4-b3d5-494c-b25c-106475fe4409" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 623.797886] env[63593]: DEBUG nova.network.neutron [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 623.814698] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.964s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 623.815323] env[63593]: ERROR nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea, please check neutron logs for more information. [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Traceback (most recent call last): [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self.driver.spawn(context, instance, image_meta, [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self._vmops.spawn(context, instance, image_meta, injected_files, [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] vm_ref = self.build_virtual_machine(instance, [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] vif_infos = vmwarevif.get_vif_info(self._session, [ 623.815323] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] for vif in network_info: [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] return self._sync_wrapper(fn, *args, **kwargs) [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self.wait() [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self[:] = self._gt.wait() [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] return self._exit_event.wait() [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] result = hub.switch() [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 623.815696] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] return self.greenlet.switch() [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] result = function(*args, **kwargs) [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] return func(*args, **kwargs) [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] raise e [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] nwinfo = self.network_api.allocate_for_instance( [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] created_port_ids = self._update_ports_for_instance( [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] with excutils.save_and_reraise_exception(): [ 623.816077] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] self.force_reraise() [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] raise self.value [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] updated_port = self._update_port( [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] _ensure_no_port_binding_failure(port) [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] raise exception.PortBindingFailed(port_id=port['id']) [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] nova.exception.PortBindingFailed: Binding failed for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea, please check neutron logs for more information. [ 623.816456] env[63593]: ERROR nova.compute.manager [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] [ 623.816780] env[63593]: DEBUG nova.compute.utils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Binding failed for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 623.817601] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 8.621s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 623.819739] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Build of instance 5bde524f-b343-423f-85b4-36cb8a6faa96 was re-scheduled: Binding failed for port 8e62cb6b-450f-4267-9f34-0b055d9af4ea, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 623.820395] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 623.820788] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Acquiring lock "refresh_cache-5bde524f-b343-423f-85b4-36cb8a6faa96" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 623.820958] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Acquired lock "refresh_cache-5bde524f-b343-423f-85b4-36cb8a6faa96" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 623.821127] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 623.896713] env[63593]: DEBUG nova.network.neutron [req-9abcb5e7-d3a0-438e-8d7e-3a19d96b74d6 req-220d6375-a72c-445c-801b-70798870cfee service nova] [instance: 70709096-265f-4531-8946-51029ebbb998] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 624.085646] env[63593]: DEBUG nova.network.neutron [req-9abcb5e7-d3a0-438e-8d7e-3a19d96b74d6 req-220d6375-a72c-445c-801b-70798870cfee service nova] [instance: 70709096-265f-4531-8946-51029ebbb998] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 624.302365] env[63593]: INFO nova.compute.manager [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] Took 1.05 seconds to deallocate network for instance. [ 624.357723] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 624.467496] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 624.507977] env[63593]: ERROR nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c5d5c11c-4a35-4e6c-bfcd-444efec65695, please check neutron logs for more information. [ 624.507977] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 624.507977] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 624.507977] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 624.507977] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 624.507977] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 624.507977] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 624.507977] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 624.507977] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 624.507977] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 624.507977] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 624.507977] env[63593]: ERROR nova.compute.manager raise self.value [ 624.507977] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 624.507977] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 624.507977] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 624.507977] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 624.508503] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 624.508503] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 624.508503] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c5d5c11c-4a35-4e6c-bfcd-444efec65695, please check neutron logs for more information. [ 624.508503] env[63593]: ERROR nova.compute.manager [ 624.508503] env[63593]: Traceback (most recent call last): [ 624.508503] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 624.508503] env[63593]: listener.cb(fileno) [ 624.508503] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 624.508503] env[63593]: result = function(*args, **kwargs) [ 624.508503] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 624.508503] env[63593]: return func(*args, **kwargs) [ 624.508503] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 624.508503] env[63593]: raise e [ 624.508503] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 624.508503] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 624.508503] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 624.508503] env[63593]: created_port_ids = self._update_ports_for_instance( [ 624.508503] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 624.508503] env[63593]: with excutils.save_and_reraise_exception(): [ 624.508503] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 624.508503] env[63593]: self.force_reraise() [ 624.508503] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 624.508503] env[63593]: raise self.value [ 624.508503] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 624.508503] env[63593]: updated_port = self._update_port( [ 624.508503] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 624.508503] env[63593]: _ensure_no_port_binding_failure(port) [ 624.508503] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 624.508503] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 624.509205] env[63593]: nova.exception.PortBindingFailed: Binding failed for port c5d5c11c-4a35-4e6c-bfcd-444efec65695, please check neutron logs for more information. [ 624.509205] env[63593]: Removing descriptor: 19 [ 624.509205] env[63593]: ERROR nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c5d5c11c-4a35-4e6c-bfcd-444efec65695, please check neutron logs for more information. [ 624.509205] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Traceback (most recent call last): [ 624.509205] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 624.509205] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] yield resources [ 624.509205] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 624.509205] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self.driver.spawn(context, instance, image_meta, [ 624.509205] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 624.509205] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 624.509205] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 624.509205] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] vm_ref = self.build_virtual_machine(instance, [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] vif_infos = vmwarevif.get_vif_info(self._session, [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] for vif in network_info: [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] return self._sync_wrapper(fn, *args, **kwargs) [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self.wait() [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self[:] = self._gt.wait() [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] return self._exit_event.wait() [ 624.509511] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] result = hub.switch() [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] return self.greenlet.switch() [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] result = function(*args, **kwargs) [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] return func(*args, **kwargs) [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] raise e [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] nwinfo = self.network_api.allocate_for_instance( [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 624.509828] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] created_port_ids = self._update_ports_for_instance( [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] with excutils.save_and_reraise_exception(): [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self.force_reraise() [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] raise self.value [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] updated_port = self._update_port( [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] _ensure_no_port_binding_failure(port) [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 624.510293] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] raise exception.PortBindingFailed(port_id=port['id']) [ 624.510613] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] nova.exception.PortBindingFailed: Binding failed for port c5d5c11c-4a35-4e6c-bfcd-444efec65695, please check neutron logs for more information. [ 624.510613] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] [ 624.510613] env[63593]: INFO nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Terminating instance [ 624.517082] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Acquiring lock "refresh_cache-d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 624.518161] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Acquired lock "refresh_cache-d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 624.518161] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 624.591546] env[63593]: DEBUG oslo_concurrency.lockutils [req-9abcb5e7-d3a0-438e-8d7e-3a19d96b74d6 req-220d6375-a72c-445c-801b-70798870cfee service nova] Releasing lock "refresh_cache-70709096-265f-4531-8946-51029ebbb998" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 624.591546] env[63593]: DEBUG nova.compute.manager [req-9abcb5e7-d3a0-438e-8d7e-3a19d96b74d6 req-220d6375-a72c-445c-801b-70798870cfee service nova] [instance: 70709096-265f-4531-8946-51029ebbb998] Received event network-vif-deleted-3ccc6572-840d-4160-95ba-8f2564a96711 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 624.858547] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 7044b799-74ed-4711-9b53-08f8a8ddbb78 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 624.970113] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Releasing lock "refresh_cache-5bde524f-b343-423f-85b4-36cb8a6faa96" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 624.970376] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 624.970547] env[63593]: DEBUG nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 624.970824] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 624.977129] env[63593]: DEBUG nova.compute.manager [req-de6c8856-217d-48b1-b254-fee58a60b672 req-96c1d856-b31f-4591-ab89-a5e7f521b41f service nova] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Received event network-changed-c5d5c11c-4a35-4e6c-bfcd-444efec65695 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 624.977319] env[63593]: DEBUG nova.compute.manager [req-de6c8856-217d-48b1-b254-fee58a60b672 req-96c1d856-b31f-4591-ab89-a5e7f521b41f service nova] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Refreshing instance network info cache due to event network-changed-c5d5c11c-4a35-4e6c-bfcd-444efec65695. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 624.977502] env[63593]: DEBUG oslo_concurrency.lockutils [req-de6c8856-217d-48b1-b254-fee58a60b672 req-96c1d856-b31f-4591-ab89-a5e7f521b41f service nova] Acquiring lock "refresh_cache-d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 625.001931] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 625.056398] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 625.144417] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "cc41b8c4-c49a-4f23-bd16-ca2362a687d0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.145261] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "cc41b8c4-c49a-4f23-bd16-ca2362a687d0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.197642] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 625.364316] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance c6cf53d2-56c8-4677-9cf9-d72d75cc922b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 625.370761] env[63593]: INFO nova.scheduler.client.report [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Deleted allocations for instance c6cf53d2-56c8-4677-9cf9-d72d75cc922b [ 625.503492] env[63593]: DEBUG nova.network.neutron [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 625.702580] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Releasing lock "refresh_cache-d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 625.703104] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 625.703510] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 625.703751] env[63593]: DEBUG oslo_concurrency.lockutils [req-de6c8856-217d-48b1-b254-fee58a60b672 req-96c1d856-b31f-4591-ab89-a5e7f521b41f service nova] Acquired lock "refresh_cache-d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 625.704560] env[63593]: DEBUG nova.network.neutron [req-de6c8856-217d-48b1-b254-fee58a60b672 req-96c1d856-b31f-4591-ab89-a5e7f521b41f service nova] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Refreshing network info cache for port c5d5c11c-4a35-4e6c-bfcd-444efec65695 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 625.707848] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c3f91036-5f6b-4866-b15f-a66f8bf4165c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.720941] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1ee1a4e-4e99-4d0e-9d59-aa1190b45c7a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.753767] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d7b23c8e-77bf-41f9-8b72-98bc660ed4c6 could not be found. [ 625.754021] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 625.754187] env[63593]: INFO nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Took 0.05 seconds to destroy the instance on the hypervisor. [ 625.754550] env[63593]: DEBUG oslo.service.loopingcall [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 625.754689] env[63593]: DEBUG nova.compute.manager [-] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 625.754739] env[63593]: DEBUG nova.network.neutron [-] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 625.794057] env[63593]: DEBUG nova.network.neutron [-] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 625.886996] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5bde524f-b343-423f-85b4-36cb8a6faa96 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 625.886996] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fe213cbb-4bed-487c-9d89-2c6b101b8956 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 625.886996] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance ed5ced72-49a9-42b5-b112-1ca01c023ae6 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 625.886996] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fa1bd569-3db7-4b96-b0b2-d6acf6ad6567 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 625.887257] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 70709096-265f-4531-8946-51029ebbb998 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 256, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 625.887257] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 625.887257] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d7b23c8e-77bf-41f9-8b72-98bc660ed4c6 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 625.890404] env[63593]: DEBUG oslo_concurrency.lockutils [None req-224e08bb-c6d0-4209-86b8-d21cdae568a5 tempest-ImagesOneServerNegativeTestJSON-119941292 tempest-ImagesOneServerNegativeTestJSON-119941292-project-member] Lock "c6cf53d2-56c8-4677-9cf9-d72d75cc922b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 29.635s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 625.891044] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "c6cf53d2-56c8-4677-9cf9-d72d75cc922b" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 24.858s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.891493] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-fcb5d1b9-c990-4983-98f4-e45f3b70adeb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.905810] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd54154a-98af-45a5-9f53-469d55cf3285 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 626.006736] env[63593]: INFO nova.compute.manager [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] [instance: 5bde524f-b343-423f-85b4-36cb8a6faa96] Took 1.04 seconds to deallocate network for instance. [ 626.124674] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Acquiring lock "c98de831-5d54-4a30-8414-9973b693c125" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.125105] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Lock "c98de831-5d54-4a30-8414-9973b693c125" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.228894] env[63593]: DEBUG nova.network.neutron [req-de6c8856-217d-48b1-b254-fee58a60b672 req-96c1d856-b31f-4591-ab89-a5e7f521b41f service nova] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 626.296403] env[63593]: DEBUG nova.network.neutron [-] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 626.311025] env[63593]: DEBUG nova.network.neutron [req-de6c8856-217d-48b1-b254-fee58a60b672 req-96c1d856-b31f-4591-ab89-a5e7f521b41f service nova] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 626.392534] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 43bcc8c6-0fd5-467b-9e11-3b788cf04236 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 626.396753] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 626.436788] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: c6cf53d2-56c8-4677-9cf9-d72d75cc922b] During the sync_power process the instance has moved from host None to host cpu-1 [ 626.437121] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "c6cf53d2-56c8-4677-9cf9-d72d75cc922b" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.546s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.800147] env[63593]: INFO nova.compute.manager [-] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Took 1.05 seconds to deallocate network for instance. [ 626.808480] env[63593]: DEBUG nova.compute.claims [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 626.808773] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.816071] env[63593]: DEBUG oslo_concurrency.lockutils [req-de6c8856-217d-48b1-b254-fee58a60b672 req-96c1d856-b31f-4591-ab89-a5e7f521b41f service nova] Releasing lock "refresh_cache-d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 626.897744] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0dbcfda1-a081-4299-a117-3a1974e2510c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 626.926774] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.054341] env[63593]: INFO nova.scheduler.client.report [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Deleted allocations for instance 5bde524f-b343-423f-85b4-36cb8a6faa96 [ 627.060142] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "9140518f-aae7-403d-acdd-8f8d80fc8552" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.060384] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "9140518f-aae7-403d-acdd-8f8d80fc8552" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.401429] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 81b0b694-b9fb-4416-8751-7f1e24e96869 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 627.566204] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2cfcd4d5-766a-4fd5-831a-783df980bbbc tempest-ServerExternalEventsTest-2113386873 tempest-ServerExternalEventsTest-2113386873-project-member] Lock "5bde524f-b343-423f-85b4-36cb8a6faa96" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 30.955s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.834329] env[63593]: DEBUG nova.compute.manager [req-a9bfffac-0543-4d7b-b558-00fa43dc6f79 req-52cf5344-dd7c-448a-8980-ca9d74dbf60a service nova] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Received event network-vif-deleted-c5d5c11c-4a35-4e6c-bfcd-444efec65695 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 627.904389] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance c6d88cbe-488b-490b-a37c-51a311276dfe has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 628.072842] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 628.409985] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 628.612556] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.917305] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance bb3a0a79-ef74-4fb0-a286-91b1b92df751 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 629.422473] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4869e3e4-48bb-446f-a15e-3a53bae1bc9b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 629.928697] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d46033b4-b3d5-494c-b25c-106475fe4409 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 630.435371] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cc41b8c4-c49a-4f23-bd16-ca2362a687d0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 630.436310] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 7 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 630.436859] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1920MB phys_disk=200GB used_disk=7GB total_vcpus=48 used_vcpus=7 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 630.750975] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7b1bf5f-4713-4898-ab06-f678db12f887 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.763783] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0eb65bc-4090-44c7-88e3-e859737abd3d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.813215] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d00a1942-f2dc-435a-9a6e-61fe14aef58a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.822531] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dec8e948-1584-4745-aea9-4bbd3ea40a1e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.839794] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 631.345333] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 631.563566] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Acquiring lock "aa57c935-efea-4d95-9fda-65fa9106c90e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 631.563843] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Lock "aa57c935-efea-4d95-9fda-65fa9106c90e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 631.855165] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 631.855165] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 8.036s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 631.855165] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.545s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 632.747539] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34915d6f-d824-45aa-bf18-abe9e3a6cea9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 632.763260] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5734c27f-d0ba-4897-8af0-03b478d22af8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 632.801679] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d5bcd6c-722a-4c84-833a-8ebd5aa404c9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 632.810500] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3cf36f70-77ed-4c6b-b670-307c516a8739 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 632.829980] env[63593]: DEBUG nova.compute.provider_tree [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 633.340707] env[63593]: DEBUG nova.scheduler.client.report [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 633.857750] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.002s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 633.857750] env[63593]: ERROR nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 39592e6f-19be-44dd-8888-4b060eb6f166, please check neutron logs for more information. [ 633.857750] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Traceback (most recent call last): [ 633.857750] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 633.857750] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self.driver.spawn(context, instance, image_meta, [ 633.857750] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 633.857750] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self._vmops.spawn(context, instance, image_meta, injected_files, [ 633.857750] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 633.857750] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] vm_ref = self.build_virtual_machine(instance, [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] vif_infos = vmwarevif.get_vif_info(self._session, [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] for vif in network_info: [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] return self._sync_wrapper(fn, *args, **kwargs) [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self.wait() [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self[:] = self._gt.wait() [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] return self._exit_event.wait() [ 633.858376] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] result = hub.switch() [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] return self.greenlet.switch() [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] result = function(*args, **kwargs) [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] return func(*args, **kwargs) [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] raise e [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] nwinfo = self.network_api.allocate_for_instance( [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 633.858803] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] created_port_ids = self._update_ports_for_instance( [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] with excutils.save_and_reraise_exception(): [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] self.force_reraise() [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] raise self.value [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] updated_port = self._update_port( [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] _ensure_no_port_binding_failure(port) [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 633.859150] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] raise exception.PortBindingFailed(port_id=port['id']) [ 633.859459] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] nova.exception.PortBindingFailed: Binding failed for port 39592e6f-19be-44dd-8888-4b060eb6f166, please check neutron logs for more information. [ 633.859459] env[63593]: ERROR nova.compute.manager [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] [ 633.859459] env[63593]: DEBUG nova.compute.utils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Binding failed for port 39592e6f-19be-44dd-8888-4b060eb6f166, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 633.859459] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.135s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 633.860492] env[63593]: INFO nova.compute.claims [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 633.863789] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Build of instance fe213cbb-4bed-487c-9d89-2c6b101b8956 was re-scheduled: Binding failed for port 39592e6f-19be-44dd-8888-4b060eb6f166, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 633.864331] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 633.864704] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "refresh_cache-fe213cbb-4bed-487c-9d89-2c6b101b8956" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 633.864902] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquired lock "refresh_cache-fe213cbb-4bed-487c-9d89-2c6b101b8956" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 633.865127] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 634.429622] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 634.594184] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 635.098091] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Releasing lock "refresh_cache-fe213cbb-4bed-487c-9d89-2c6b101b8956" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 635.098091] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 635.098992] env[63593]: DEBUG nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 635.098992] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 635.127928] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 635.240457] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c97bc1c-2098-46bf-91c5-e4b8b32361d6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.250793] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5fdd8b8e-fbf1-4563-a1df-5e3b1e8f56c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.299317] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2dbe734a-14ea-4107-bec4-4d709232b52a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.308233] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bfda214-cce4-4e1a-8aee-e1568d100eef {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.324299] env[63593]: DEBUG nova.compute.provider_tree [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 635.633325] env[63593]: DEBUG nova.network.neutron [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 635.828834] env[63593]: DEBUG nova.scheduler.client.report [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 636.141848] env[63593]: INFO nova.compute.manager [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: fe213cbb-4bed-487c-9d89-2c6b101b8956] Took 1.04 seconds to deallocate network for instance. [ 636.339928] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.479s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.339928] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 636.341893] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.771s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.855019] env[63593]: DEBUG nova.compute.utils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 636.860858] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 636.861062] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 637.056283] env[63593]: DEBUG nova.policy [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4f8e00cc2ef6468ca21134955c704b27', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '4f60525c71c74dd68c3cdaa50fe19deb', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 637.186743] env[63593]: INFO nova.scheduler.client.report [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Deleted allocations for instance fe213cbb-4bed-487c-9d89-2c6b101b8956 [ 637.228645] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d759d47a-23cd-4a8c-ae3d-4a5be890d2d9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.239712] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8464e0e-0a97-4aac-99c9-fa13e651fe6b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.279302] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b11d8bb1-6b73-42d3-8a0d-0e158ea62774 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.307678] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c25a47cb-1669-4f83-879d-a5f9d8233cde {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.324629] env[63593]: DEBUG nova.compute.provider_tree [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 637.361474] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 637.698046] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2ff0c0f6-de6d-4d1d-93f2-8524d0187317 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "fe213cbb-4bed-487c-9d89-2c6b101b8956" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 39.322s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.828397] env[63593]: DEBUG nova.scheduler.client.report [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 638.201873] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 638.336276] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.994s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 638.336795] env[63593]: ERROR nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 56519d14-008a-4cfe-a540-50b419d4570d, please check neutron logs for more information. [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Traceback (most recent call last): [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self.driver.spawn(context, instance, image_meta, [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] vm_ref = self.build_virtual_machine(instance, [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] vif_infos = vmwarevif.get_vif_info(self._session, [ 638.336795] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] for vif in network_info: [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] return self._sync_wrapper(fn, *args, **kwargs) [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self.wait() [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self[:] = self._gt.wait() [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] return self._exit_event.wait() [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] result = hub.switch() [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 638.337150] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] return self.greenlet.switch() [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] result = function(*args, **kwargs) [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] return func(*args, **kwargs) [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] raise e [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] nwinfo = self.network_api.allocate_for_instance( [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] created_port_ids = self._update_ports_for_instance( [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] with excutils.save_and_reraise_exception(): [ 638.337478] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] self.force_reraise() [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] raise self.value [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] updated_port = self._update_port( [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] _ensure_no_port_binding_failure(port) [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] raise exception.PortBindingFailed(port_id=port['id']) [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] nova.exception.PortBindingFailed: Binding failed for port 56519d14-008a-4cfe-a540-50b419d4570d, please check neutron logs for more information. [ 638.337856] env[63593]: ERROR nova.compute.manager [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] [ 638.338169] env[63593]: DEBUG nova.compute.utils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Binding failed for port 56519d14-008a-4cfe-a540-50b419d4570d, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 638.338737] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.752s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 638.348078] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Build of instance ed5ced72-49a9-42b5-b112-1ca01c023ae6 was re-scheduled: Binding failed for port 56519d14-008a-4cfe-a540-50b419d4570d, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 638.349948] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 638.349948] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "refresh_cache-ed5ced72-49a9-42b5-b112-1ca01c023ae6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 638.349948] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquired lock "refresh_cache-ed5ced72-49a9-42b5-b112-1ca01c023ae6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 638.349948] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 638.373365] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 638.428584] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 638.428812] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 638.428960] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 638.429163] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 638.429307] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 638.433347] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 638.433598] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 638.433762] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 638.433939] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 638.434249] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 638.434775] env[63593]: DEBUG nova.virt.hardware [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 638.435637] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f74ad9a2-645b-4021-acfd-653a771568c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.449220] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-098e7795-9b0a-4500-99c5-385f6f7caef4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.739117] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 638.836417] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Successfully created port: 964fa279-e5ea-4fa0-b942-120129c86d59 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 638.956519] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 639.184659] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea08a4f6-2494-4288-9bd4-d092029b03c3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.201726] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7cdecb1c-e763-46f3-a6f1-7fe36fdfb547 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.242221] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad34f755-b399-46b1-ae0d-6d0e11ec22d7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.252750] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-643f7da7-b9ca-4755-8dd4-1191bbae12e1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.269089] env[63593]: DEBUG nova.compute.provider_tree [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 639.342145] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 639.772108] env[63593]: DEBUG nova.scheduler.client.report [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 639.851236] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Releasing lock "refresh_cache-ed5ced72-49a9-42b5-b112-1ca01c023ae6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 639.851236] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 639.851236] env[63593]: DEBUG nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 639.851236] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 639.879271] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 640.141953] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Acquiring lock "6192c09e-64e7-4eb3-968e-922247892ca2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 640.142254] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Lock "6192c09e-64e7-4eb3-968e-922247892ca2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 640.283522] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.944s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 640.284310] env[63593]: ERROR nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 27bc5beb-cb5f-470e-91ff-918686f9a252, please check neutron logs for more information. [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Traceback (most recent call last): [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self.driver.spawn(context, instance, image_meta, [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self._vmops.spawn(context, instance, image_meta, injected_files, [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] vm_ref = self.build_virtual_machine(instance, [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] vif_infos = vmwarevif.get_vif_info(self._session, [ 640.284310] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] for vif in network_info: [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] return self._sync_wrapper(fn, *args, **kwargs) [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self.wait() [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self[:] = self._gt.wait() [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] return self._exit_event.wait() [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] result = hub.switch() [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 640.284671] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] return self.greenlet.switch() [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] result = function(*args, **kwargs) [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] return func(*args, **kwargs) [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] raise e [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] nwinfo = self.network_api.allocate_for_instance( [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] created_port_ids = self._update_ports_for_instance( [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] with excutils.save_and_reraise_exception(): [ 640.285103] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] self.force_reraise() [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] raise self.value [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] updated_port = self._update_port( [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] _ensure_no_port_binding_failure(port) [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] raise exception.PortBindingFailed(port_id=port['id']) [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] nova.exception.PortBindingFailed: Binding failed for port 27bc5beb-cb5f-470e-91ff-918686f9a252, please check neutron logs for more information. [ 640.285431] env[63593]: ERROR nova.compute.manager [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] [ 640.286363] env[63593]: DEBUG nova.compute.utils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Binding failed for port 27bc5beb-cb5f-470e-91ff-918686f9a252, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 640.288115] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.776s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 640.293131] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Build of instance fa1bd569-3db7-4b96-b0b2-d6acf6ad6567 was re-scheduled: Binding failed for port 27bc5beb-cb5f-470e-91ff-918686f9a252, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 640.293710] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 640.294011] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "refresh_cache-fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 640.294422] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquired lock "refresh_cache-fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 640.294493] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 640.385663] env[63593]: DEBUG nova.network.neutron [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 640.466594] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Acquiring lock "1205d83c-163f-41a4-ab32-a5a706763b33" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 640.466594] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Lock "1205d83c-163f-41a4-ab32-a5a706763b33" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 640.830158] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 640.894024] env[63593]: INFO nova.compute.manager [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: ed5ced72-49a9-42b5-b112-1ca01c023ae6] Took 1.04 seconds to deallocate network for instance. [ 640.959944] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 641.192272] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf9cf5d1-535f-49d7-92df-e52afb6a4dd3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.201424] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab510908-707f-4ac0-be63-628b4f570d18 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.238295] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14d225b7-5293-4641-be73-ba724946c41f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.247719] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97bea15d-9eb7-4341-9e56-4cd8346bc990 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.263028] env[63593]: DEBUG nova.compute.provider_tree [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 641.464431] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Releasing lock "refresh_cache-fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 641.464741] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 641.466039] env[63593]: DEBUG nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 641.466039] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 641.490611] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 641.766431] env[63593]: DEBUG nova.scheduler.client.report [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 641.935111] env[63593]: INFO nova.scheduler.client.report [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Deleted allocations for instance ed5ced72-49a9-42b5-b112-1ca01c023ae6 [ 641.995217] env[63593]: DEBUG nova.network.neutron [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 642.102483] env[63593]: ERROR nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 964fa279-e5ea-4fa0-b942-120129c86d59, please check neutron logs for more information. [ 642.102483] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 642.102483] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 642.102483] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 642.102483] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 642.102483] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 642.102483] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 642.102483] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 642.102483] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 642.102483] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 642.102483] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 642.102483] env[63593]: ERROR nova.compute.manager raise self.value [ 642.102483] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 642.102483] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 642.102483] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 642.102483] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 642.103041] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 642.103041] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 642.103041] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 964fa279-e5ea-4fa0-b942-120129c86d59, please check neutron logs for more information. [ 642.103041] env[63593]: ERROR nova.compute.manager [ 642.103041] env[63593]: Traceback (most recent call last): [ 642.103041] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 642.103041] env[63593]: listener.cb(fileno) [ 642.103041] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 642.103041] env[63593]: result = function(*args, **kwargs) [ 642.103041] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 642.103041] env[63593]: return func(*args, **kwargs) [ 642.103041] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 642.103041] env[63593]: raise e [ 642.103041] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 642.103041] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 642.103041] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 642.103041] env[63593]: created_port_ids = self._update_ports_for_instance( [ 642.103041] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 642.103041] env[63593]: with excutils.save_and_reraise_exception(): [ 642.103041] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 642.103041] env[63593]: self.force_reraise() [ 642.103041] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 642.103041] env[63593]: raise self.value [ 642.103041] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 642.103041] env[63593]: updated_port = self._update_port( [ 642.103041] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 642.103041] env[63593]: _ensure_no_port_binding_failure(port) [ 642.103041] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 642.103041] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 642.103966] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 964fa279-e5ea-4fa0-b942-120129c86d59, please check neutron logs for more information. [ 642.103966] env[63593]: Removing descriptor: 19 [ 642.103966] env[63593]: ERROR nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 964fa279-e5ea-4fa0-b942-120129c86d59, please check neutron logs for more information. [ 642.103966] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Traceback (most recent call last): [ 642.103966] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 642.103966] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] yield resources [ 642.103966] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 642.103966] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self.driver.spawn(context, instance, image_meta, [ 642.103966] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 642.103966] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self._vmops.spawn(context, instance, image_meta, injected_files, [ 642.103966] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 642.103966] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] vm_ref = self.build_virtual_machine(instance, [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] vif_infos = vmwarevif.get_vif_info(self._session, [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] for vif in network_info: [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] return self._sync_wrapper(fn, *args, **kwargs) [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self.wait() [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self[:] = self._gt.wait() [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] return self._exit_event.wait() [ 642.105205] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] result = hub.switch() [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] return self.greenlet.switch() [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] result = function(*args, **kwargs) [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] return func(*args, **kwargs) [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] raise e [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] nwinfo = self.network_api.allocate_for_instance( [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 642.105715] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] created_port_ids = self._update_ports_for_instance( [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] with excutils.save_and_reraise_exception(): [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self.force_reraise() [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] raise self.value [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] updated_port = self._update_port( [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] _ensure_no_port_binding_failure(port) [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 642.106146] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] raise exception.PortBindingFailed(port_id=port['id']) [ 642.106683] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] nova.exception.PortBindingFailed: Binding failed for port 964fa279-e5ea-4fa0-b942-120129c86d59, please check neutron logs for more information. [ 642.106683] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] [ 642.106683] env[63593]: INFO nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Terminating instance [ 642.106683] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Acquiring lock "refresh_cache-43bcc8c6-0fd5-467b-9e11-3b788cf04236" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 642.106683] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Acquired lock "refresh_cache-43bcc8c6-0fd5-467b-9e11-3b788cf04236" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 642.106683] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 642.273537] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.985s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 642.275861] env[63593]: ERROR nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3ccc6572-840d-4160-95ba-8f2564a96711, please check neutron logs for more information. [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] Traceback (most recent call last): [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self.driver.spawn(context, instance, image_meta, [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self._vmops.spawn(context, instance, image_meta, injected_files, [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] vm_ref = self.build_virtual_machine(instance, [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] vif_infos = vmwarevif.get_vif_info(self._session, [ 642.275861] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] for vif in network_info: [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] return self._sync_wrapper(fn, *args, **kwargs) [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self.wait() [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self[:] = self._gt.wait() [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] return self._exit_event.wait() [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] result = hub.switch() [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 642.276475] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] return self.greenlet.switch() [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] result = function(*args, **kwargs) [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] return func(*args, **kwargs) [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] raise e [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] nwinfo = self.network_api.allocate_for_instance( [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] created_port_ids = self._update_ports_for_instance( [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] with excutils.save_and_reraise_exception(): [ 642.277072] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] self.force_reraise() [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] raise self.value [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] updated_port = self._update_port( [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] _ensure_no_port_binding_failure(port) [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] raise exception.PortBindingFailed(port_id=port['id']) [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] nova.exception.PortBindingFailed: Binding failed for port 3ccc6572-840d-4160-95ba-8f2564a96711, please check neutron logs for more information. [ 642.277828] env[63593]: ERROR nova.compute.manager [instance: 70709096-265f-4531-8946-51029ebbb998] [ 642.278350] env[63593]: DEBUG nova.compute.utils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Binding failed for port 3ccc6572-840d-4160-95ba-8f2564a96711, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 642.278350] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.468s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.281026] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Build of instance 70709096-265f-4531-8946-51029ebbb998 was re-scheduled: Binding failed for port 3ccc6572-840d-4160-95ba-8f2564a96711, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 642.281256] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 642.281555] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquiring lock "refresh_cache-70709096-265f-4531-8946-51029ebbb998" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 642.281830] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Acquired lock "refresh_cache-70709096-265f-4531-8946-51029ebbb998" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 642.281830] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 642.416093] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Acquiring lock "6bf23f6a-6727-448f-80d9-fe954addd49e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.416349] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Lock "6bf23f6a-6727-448f-80d9-fe954addd49e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.434353] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "6462167e-ae83-4d73-bfb2-ec7024e56a07" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.434589] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "6462167e-ae83-4d73-bfb2-ec7024e56a07" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.449084] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7a3f9fc8-1258-470e-bdf7-281c75811c1e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "ed5ced72-49a9-42b5-b112-1ca01c023ae6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 43.389s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 642.502172] env[63593]: INFO nova.compute.manager [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: fa1bd569-3db7-4b96-b0b2-d6acf6ad6567] Took 1.03 seconds to deallocate network for instance. [ 642.631687] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 642.690284] env[63593]: DEBUG nova.compute.manager [req-719a0a33-9376-4dce-b0ad-44b6ed875788 req-c5015714-554f-458f-b536-bc978d4b542b service nova] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Received event network-changed-964fa279-e5ea-4fa0-b942-120129c86d59 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 642.690481] env[63593]: DEBUG nova.compute.manager [req-719a0a33-9376-4dce-b0ad-44b6ed875788 req-c5015714-554f-458f-b536-bc978d4b542b service nova] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Refreshing instance network info cache due to event network-changed-964fa279-e5ea-4fa0-b942-120129c86d59. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 642.690702] env[63593]: DEBUG oslo_concurrency.lockutils [req-719a0a33-9376-4dce-b0ad-44b6ed875788 req-c5015714-554f-458f-b536-bc978d4b542b service nova] Acquiring lock "refresh_cache-43bcc8c6-0fd5-467b-9e11-3b788cf04236" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 642.742788] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 642.817049] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 642.953481] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 642.980302] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 643.192585] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97243e53-57e6-4bb8-89f7-10aa7a57b409 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.202760] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02772849-0cd0-41d0-a9f3-9606cbd715c5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.241067] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ddc77f4-de39-4196-aa90-73ef9f1ad565 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.247542] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Releasing lock "refresh_cache-43bcc8c6-0fd5-467b-9e11-3b788cf04236" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 643.247542] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 643.247946] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 643.250583] env[63593]: DEBUG oslo_concurrency.lockutils [req-719a0a33-9376-4dce-b0ad-44b6ed875788 req-c5015714-554f-458f-b536-bc978d4b542b service nova] Acquired lock "refresh_cache-43bcc8c6-0fd5-467b-9e11-3b788cf04236" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 643.250789] env[63593]: DEBUG nova.network.neutron [req-719a0a33-9376-4dce-b0ad-44b6ed875788 req-c5015714-554f-458f-b536-bc978d4b542b service nova] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Refreshing network info cache for port 964fa279-e5ea-4fa0-b942-120129c86d59 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 643.251948] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9f14c993-eec6-4ac7-88aa-7e42fa45fd59 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.257019] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a17c36e7-b498-49f2-99a3-1fa04a441422 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.268784] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ac411c4-88ce-4e13-a469-47721d521257 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.294777] env[63593]: DEBUG nova.compute.provider_tree [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 643.301475] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 43bcc8c6-0fd5-467b-9e11-3b788cf04236 could not be found. [ 643.301738] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 643.302066] env[63593]: INFO nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Took 0.05 seconds to destroy the instance on the hypervisor. [ 643.302410] env[63593]: DEBUG oslo.service.loopingcall [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 643.303767] env[63593]: DEBUG nova.compute.manager [-] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 643.303767] env[63593]: DEBUG nova.network.neutron [-] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 643.337603] env[63593]: DEBUG nova.network.neutron [-] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 643.484883] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 643.485388] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Releasing lock "refresh_cache-70709096-265f-4531-8946-51029ebbb998" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 643.485622] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 643.485797] env[63593]: DEBUG nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 643.486015] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 643.501944] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 643.544508] env[63593]: INFO nova.scheduler.client.report [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Deleted allocations for instance fa1bd569-3db7-4b96-b0b2-d6acf6ad6567 [ 643.774519] env[63593]: DEBUG nova.network.neutron [req-719a0a33-9376-4dce-b0ad-44b6ed875788 req-c5015714-554f-458f-b536-bc978d4b542b service nova] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 643.804396] env[63593]: DEBUG nova.scheduler.client.report [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 643.831964] env[63593]: DEBUG nova.network.neutron [req-719a0a33-9376-4dce-b0ad-44b6ed875788 req-c5015714-554f-458f-b536-bc978d4b542b service nova] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 643.839912] env[63593]: DEBUG nova.network.neutron [-] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 644.005574] env[63593]: DEBUG nova.network.neutron [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 644.055069] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c19561b8-7f65-4e15-aff7-0e6e79de547e tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "fa1bd569-3db7-4b96-b0b2-d6acf6ad6567" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 43.355s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 644.189433] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "04de495d-8896-4c6d-a059-9a80477e3b07" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.189726] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "04de495d-8896-4c6d-a059-9a80477e3b07" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 644.311418] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.032s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 644.311418] env[63593]: ERROR nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c5d5c11c-4a35-4e6c-bfcd-444efec65695, please check neutron logs for more information. [ 644.311418] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Traceback (most recent call last): [ 644.311418] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 644.311418] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self.driver.spawn(context, instance, image_meta, [ 644.311418] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 644.311418] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 644.311418] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 644.311418] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] vm_ref = self.build_virtual_machine(instance, [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] vif_infos = vmwarevif.get_vif_info(self._session, [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] for vif in network_info: [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] return self._sync_wrapper(fn, *args, **kwargs) [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self.wait() [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self[:] = self._gt.wait() [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] return self._exit_event.wait() [ 644.311757] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] result = hub.switch() [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] return self.greenlet.switch() [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] result = function(*args, **kwargs) [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] return func(*args, **kwargs) [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] raise e [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] nwinfo = self.network_api.allocate_for_instance( [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 644.312114] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] created_port_ids = self._update_ports_for_instance( [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] with excutils.save_and_reraise_exception(): [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] self.force_reraise() [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] raise self.value [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] updated_port = self._update_port( [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] _ensure_no_port_binding_failure(port) [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 644.312446] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] raise exception.PortBindingFailed(port_id=port['id']) [ 644.312750] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] nova.exception.PortBindingFailed: Binding failed for port c5d5c11c-4a35-4e6c-bfcd-444efec65695, please check neutron logs for more information. [ 644.312750] env[63593]: ERROR nova.compute.manager [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] [ 644.312750] env[63593]: DEBUG nova.compute.utils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Binding failed for port c5d5c11c-4a35-4e6c-bfcd-444efec65695, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 644.312750] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.385s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 644.313590] env[63593]: INFO nova.compute.claims [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 644.318233] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Build of instance d7b23c8e-77bf-41f9-8b72-98bc660ed4c6 was re-scheduled: Binding failed for port c5d5c11c-4a35-4e6c-bfcd-444efec65695, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 644.318712] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 644.322020] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Acquiring lock "refresh_cache-d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 644.322020] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Acquired lock "refresh_cache-d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 644.322020] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 644.334468] env[63593]: DEBUG oslo_concurrency.lockutils [req-719a0a33-9376-4dce-b0ad-44b6ed875788 req-c5015714-554f-458f-b536-bc978d4b542b service nova] Releasing lock "refresh_cache-43bcc8c6-0fd5-467b-9e11-3b788cf04236" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 644.342604] env[63593]: INFO nova.compute.manager [-] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Took 1.04 seconds to deallocate network for instance. [ 644.345258] env[63593]: DEBUG nova.compute.claims [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 644.346119] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.509108] env[63593]: INFO nova.compute.manager [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] [instance: 70709096-265f-4531-8946-51029ebbb998] Took 1.02 seconds to deallocate network for instance. [ 644.556101] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 644.937508] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 645.040173] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 645.051265] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 645.051474] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 645.080813] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 645.463854] env[63593]: DEBUG nova.compute.manager [req-62002c88-cc92-4a1f-879a-82ebb0f5e813 req-5d5722bc-b0b4-4aa6-89a6-91677cb54b81 service nova] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Received event network-vif-deleted-964fa279-e5ea-4fa0-b942-120129c86d59 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 645.548723] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Releasing lock "refresh_cache-d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 645.548955] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 645.553297] env[63593]: DEBUG nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 645.553297] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 645.553780] env[63593]: INFO nova.scheduler.client.report [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Deleted allocations for instance 70709096-265f-4531-8946-51029ebbb998 [ 645.609104] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 645.772016] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-301a0171-9672-40f4-a8bc-102206e6c8e9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.787112] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a196a151-242d-4f3b-bb78-0d2018fec741 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.829108] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ffee386-b9cf-4a3b-9e01-6aefd1029c6d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.841594] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d190360b-7e46-4d4b-99a7-8f412ded6cd7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.859170] env[63593]: DEBUG nova.compute.provider_tree [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 645.927795] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Acquiring lock "83204968-ba8d-4e40-9f3b-d6d427fc5bb5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 645.927795] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Lock "83204968-ba8d-4e40-9f3b-d6d427fc5bb5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.065185] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b9ce1b15-c252-4632-8650-d592d03f5fdb tempest-ListServerFiltersTestJSON-1844427388 tempest-ListServerFiltersTestJSON-1844427388-project-member] Lock "70709096-265f-4531-8946-51029ebbb998" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 44.150s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.114527] env[63593]: DEBUG nova.network.neutron [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 646.365117] env[63593]: DEBUG nova.scheduler.client.report [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 646.567867] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 646.618520] env[63593]: INFO nova.compute.manager [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] [instance: d7b23c8e-77bf-41f9-8b72-98bc660ed4c6] Took 1.07 seconds to deallocate network for instance. [ 646.874068] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.562s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.874860] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 646.882339] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.270s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.884692] env[63593]: INFO nova.compute.claims [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 647.103409] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.394094] env[63593]: DEBUG nova.compute.utils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 647.399611] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 647.399907] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 647.519716] env[63593]: DEBUG nova.policy [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7f4a92be9fe846a581a2881f8344eacc', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ec2372701f1844d1ba1364fc7b821d13', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 647.657912] env[63593]: INFO nova.scheduler.client.report [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Deleted allocations for instance d7b23c8e-77bf-41f9-8b72-98bc660ed4c6 [ 647.900698] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 647.980765] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Acquiring lock "40449f7d-3a21-487a-a778-43ccd7b23949" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.980765] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Lock "40449f7d-3a21-487a-a778-43ccd7b23949" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.041981] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Successfully created port: 34edec86-cf21-4cd5-8dfd-219a396ba46b {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 648.173085] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34c3981e-b421-4801-9a9c-4f706a1ee011 tempest-ServersAdminNegativeTestJSON-1013917104 tempest-ServersAdminNegativeTestJSON-1013917104-project-member] Lock "d7b23c8e-77bf-41f9-8b72-98bc660ed4c6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 38.054s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 648.312680] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abf9f449-042e-4026-a619-1a75e71308a9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.325497] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55104028-c9eb-4d32-9628-2a2a7d323a05 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.359018] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b62386ea-d48b-4c2a-8bf1-d48e1fe86d02 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.367776] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a552507-0aeb-4c00-ba4b-f23b18601b0c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.389097] env[63593]: DEBUG nova.compute.provider_tree [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 648.679561] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 648.891260] env[63593]: DEBUG nova.scheduler.client.report [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 648.912084] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 648.942591] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 648.942945] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 648.943365] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 648.944012] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 648.944253] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 648.944448] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 648.944899] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 648.945157] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 648.945401] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 648.945610] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 648.949271] env[63593]: DEBUG nova.virt.hardware [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 648.950456] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ddc3fa7-f0ef-404a-af44-de04ffb81cd3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.963407] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a728323-4b68-46a2-bcdd-9cb9ef6a660b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.217661] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 649.397387] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.515s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 649.397899] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 649.403031] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.664s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 649.404245] env[63593]: INFO nova.compute.claims [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 649.913306] env[63593]: DEBUG nova.compute.utils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 649.920571] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 649.923210] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 649.978691] env[63593]: DEBUG nova.policy [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd014d8e414164f7ab9fafae7e2d6f73e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '14d96cd638ed4c2abab7fb07b487acd4', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 650.412764] env[63593]: ERROR nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 34edec86-cf21-4cd5-8dfd-219a396ba46b, please check neutron logs for more information. [ 650.412764] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 650.412764] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 650.412764] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 650.412764] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 650.412764] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 650.412764] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 650.412764] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 650.412764] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 650.412764] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 650.412764] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 650.412764] env[63593]: ERROR nova.compute.manager raise self.value [ 650.412764] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 650.412764] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 650.412764] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 650.412764] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 650.413213] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 650.413213] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 650.413213] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 34edec86-cf21-4cd5-8dfd-219a396ba46b, please check neutron logs for more information. [ 650.413213] env[63593]: ERROR nova.compute.manager [ 650.413426] env[63593]: Traceback (most recent call last): [ 650.413426] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 650.413426] env[63593]: listener.cb(fileno) [ 650.413426] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 650.413426] env[63593]: result = function(*args, **kwargs) [ 650.413426] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 650.413426] env[63593]: return func(*args, **kwargs) [ 650.413426] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 650.413426] env[63593]: raise e [ 650.413426] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 650.413426] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 650.413426] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 650.413426] env[63593]: created_port_ids = self._update_ports_for_instance( [ 650.413426] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 650.413426] env[63593]: with excutils.save_and_reraise_exception(): [ 650.413426] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 650.413426] env[63593]: self.force_reraise() [ 650.413426] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 650.413426] env[63593]: raise self.value [ 650.413928] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 650.413928] env[63593]: updated_port = self._update_port( [ 650.413928] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 650.413928] env[63593]: _ensure_no_port_binding_failure(port) [ 650.413928] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 650.413928] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 650.413928] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 34edec86-cf21-4cd5-8dfd-219a396ba46b, please check neutron logs for more information. [ 650.413928] env[63593]: Removing descriptor: 19 [ 650.416275] env[63593]: ERROR nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 34edec86-cf21-4cd5-8dfd-219a396ba46b, please check neutron logs for more information. [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Traceback (most recent call last): [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] yield resources [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self.driver.spawn(context, instance, image_meta, [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] vm_ref = self.build_virtual_machine(instance, [ 650.416275] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] vif_infos = vmwarevif.get_vif_info(self._session, [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] for vif in network_info: [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] return self._sync_wrapper(fn, *args, **kwargs) [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self.wait() [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self[:] = self._gt.wait() [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] return self._exit_event.wait() [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 650.417736] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] result = hub.switch() [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] return self.greenlet.switch() [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] result = function(*args, **kwargs) [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] return func(*args, **kwargs) [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] raise e [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] nwinfo = self.network_api.allocate_for_instance( [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] created_port_ids = self._update_ports_for_instance( [ 650.418520] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] with excutils.save_and_reraise_exception(): [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self.force_reraise() [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] raise self.value [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] updated_port = self._update_port( [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] _ensure_no_port_binding_failure(port) [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] raise exception.PortBindingFailed(port_id=port['id']) [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] nova.exception.PortBindingFailed: Binding failed for port 34edec86-cf21-4cd5-8dfd-219a396ba46b, please check neutron logs for more information. [ 650.419175] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] [ 650.419524] env[63593]: INFO nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Terminating instance [ 650.419722] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Acquiring lock "refresh_cache-0dbcfda1-a081-4299-a117-3a1974e2510c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 650.419973] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Acquired lock "refresh_cache-0dbcfda1-a081-4299-a117-3a1974e2510c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 650.420255] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 650.424299] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 650.453324] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Successfully created port: c5eddc15-1383-4b48-be39-d5589ce77368 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 650.792762] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f166e74e-6214-46e7-a109-8ac972f62096 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.801212] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8b96bbb-1b6f-463f-ab8e-eca2e243e75e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.837474] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-627b0acf-d6fc-46dc-a406-13893719538e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.847023] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d16a53b2-52f4-4a4b-9f29-65acec7756b3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.860101] env[63593]: DEBUG nova.compute.provider_tree [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 650.940497] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 651.058252] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 651.114833] env[63593]: DEBUG nova.compute.manager [req-859a105d-504d-461e-866c-6c6417da17ca req-fff53332-990e-4f63-82cf-5afa762a4b5f service nova] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Received event network-changed-34edec86-cf21-4cd5-8dfd-219a396ba46b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 651.114833] env[63593]: DEBUG nova.compute.manager [req-859a105d-504d-461e-866c-6c6417da17ca req-fff53332-990e-4f63-82cf-5afa762a4b5f service nova] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Refreshing instance network info cache due to event network-changed-34edec86-cf21-4cd5-8dfd-219a396ba46b. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 651.114833] env[63593]: DEBUG oslo_concurrency.lockutils [req-859a105d-504d-461e-866c-6c6417da17ca req-fff53332-990e-4f63-82cf-5afa762a4b5f service nova] Acquiring lock "refresh_cache-0dbcfda1-a081-4299-a117-3a1974e2510c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 651.365543] env[63593]: DEBUG nova.scheduler.client.report [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 651.444438] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 651.475983] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 651.476289] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 651.476447] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 651.476624] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 651.477144] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 651.477344] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 651.477561] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 651.477723] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 651.477891] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 651.479350] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 651.479350] env[63593]: DEBUG nova.virt.hardware [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 651.479771] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7fa253e-db56-4a5f-a705-f84a0db640c0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.489508] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b6f555d-f969-4865-8264-2e3cbbae47e8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.562162] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Releasing lock "refresh_cache-0dbcfda1-a081-4299-a117-3a1974e2510c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 651.565143] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 651.565614] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 651.566022] env[63593]: DEBUG oslo_concurrency.lockutils [req-859a105d-504d-461e-866c-6c6417da17ca req-fff53332-990e-4f63-82cf-5afa762a4b5f service nova] Acquired lock "refresh_cache-0dbcfda1-a081-4299-a117-3a1974e2510c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 651.566022] env[63593]: DEBUG nova.network.neutron [req-859a105d-504d-461e-866c-6c6417da17ca req-fff53332-990e-4f63-82cf-5afa762a4b5f service nova] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Refreshing network info cache for port 34edec86-cf21-4cd5-8dfd-219a396ba46b {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 651.567200] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b027a270-4fa0-42ad-a645-f6d651df7ba7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.582026] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6fdd94fc-ab0b-49e0-8c89-c732579ae930 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.609922] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0dbcfda1-a081-4299-a117-3a1974e2510c could not be found. [ 651.610204] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 651.611255] env[63593]: INFO nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Took 0.05 seconds to destroy the instance on the hypervisor. [ 651.611255] env[63593]: DEBUG oslo.service.loopingcall [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 651.611255] env[63593]: DEBUG nova.compute.manager [-] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 651.611255] env[63593]: DEBUG nova.network.neutron [-] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 651.627279] env[63593]: DEBUG nova.network.neutron [-] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 651.872477] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.470s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 651.873015] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 651.877422] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 8.392s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 651.878959] env[63593]: INFO nova.compute.claims [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 652.090087] env[63593]: DEBUG nova.network.neutron [req-859a105d-504d-461e-866c-6c6417da17ca req-fff53332-990e-4f63-82cf-5afa762a4b5f service nova] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 652.132375] env[63593]: DEBUG nova.network.neutron [-] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 652.213029] env[63593]: DEBUG nova.network.neutron [req-859a105d-504d-461e-866c-6c6417da17ca req-fff53332-990e-4f63-82cf-5afa762a4b5f service nova] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 652.379426] env[63593]: DEBUG nova.compute.utils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 652.382036] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 652.383994] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 652.458710] env[63593]: DEBUG nova.policy [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4cde8717c5284a3fa02ccac456702710', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'fb9e40fdb0fa485d81d404850b75bacb', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 652.635642] env[63593]: INFO nova.compute.manager [-] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Took 1.02 seconds to deallocate network for instance. [ 652.638987] env[63593]: DEBUG nova.compute.claims [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 652.638987] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 652.715749] env[63593]: DEBUG oslo_concurrency.lockutils [req-859a105d-504d-461e-866c-6c6417da17ca req-fff53332-990e-4f63-82cf-5afa762a4b5f service nova] Releasing lock "refresh_cache-0dbcfda1-a081-4299-a117-3a1974e2510c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 652.809272] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Successfully created port: 289407e8-c816-4ada-a146-53327b567f6c {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 652.886474] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 652.921130] env[63593]: ERROR nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c5eddc15-1383-4b48-be39-d5589ce77368, please check neutron logs for more information. [ 652.921130] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 652.921130] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 652.921130] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 652.921130] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 652.921130] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 652.921130] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 652.921130] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 652.921130] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 652.921130] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 652.921130] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 652.921130] env[63593]: ERROR nova.compute.manager raise self.value [ 652.921130] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 652.921130] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 652.921130] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 652.921130] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 652.921723] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 652.921723] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 652.921723] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c5eddc15-1383-4b48-be39-d5589ce77368, please check neutron logs for more information. [ 652.921723] env[63593]: ERROR nova.compute.manager [ 652.921723] env[63593]: Traceback (most recent call last): [ 652.921723] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 652.921723] env[63593]: listener.cb(fileno) [ 652.921723] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 652.921723] env[63593]: result = function(*args, **kwargs) [ 652.921723] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 652.921723] env[63593]: return func(*args, **kwargs) [ 652.921723] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 652.921723] env[63593]: raise e [ 652.921723] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 652.921723] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 652.921723] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 652.921723] env[63593]: created_port_ids = self._update_ports_for_instance( [ 652.921723] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 652.921723] env[63593]: with excutils.save_and_reraise_exception(): [ 652.921723] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 652.921723] env[63593]: self.force_reraise() [ 652.921723] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 652.921723] env[63593]: raise self.value [ 652.921723] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 652.921723] env[63593]: updated_port = self._update_port( [ 652.921723] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 652.921723] env[63593]: _ensure_no_port_binding_failure(port) [ 652.921723] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 652.921723] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 652.922694] env[63593]: nova.exception.PortBindingFailed: Binding failed for port c5eddc15-1383-4b48-be39-d5589ce77368, please check neutron logs for more information. [ 652.922694] env[63593]: Removing descriptor: 21 [ 652.922694] env[63593]: ERROR nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c5eddc15-1383-4b48-be39-d5589ce77368, please check neutron logs for more information. [ 652.922694] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Traceback (most recent call last): [ 652.922694] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 652.922694] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] yield resources [ 652.922694] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 652.922694] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self.driver.spawn(context, instance, image_meta, [ 652.922694] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 652.922694] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self._vmops.spawn(context, instance, image_meta, injected_files, [ 652.922694] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 652.922694] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] vm_ref = self.build_virtual_machine(instance, [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] vif_infos = vmwarevif.get_vif_info(self._session, [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] for vif in network_info: [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] return self._sync_wrapper(fn, *args, **kwargs) [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self.wait() [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self[:] = self._gt.wait() [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] return self._exit_event.wait() [ 652.923209] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] result = hub.switch() [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] return self.greenlet.switch() [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] result = function(*args, **kwargs) [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] return func(*args, **kwargs) [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] raise e [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] nwinfo = self.network_api.allocate_for_instance( [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 652.923735] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] created_port_ids = self._update_ports_for_instance( [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] with excutils.save_and_reraise_exception(): [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self.force_reraise() [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] raise self.value [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] updated_port = self._update_port( [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] _ensure_no_port_binding_failure(port) [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 652.924181] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] raise exception.PortBindingFailed(port_id=port['id']) [ 652.924633] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] nova.exception.PortBindingFailed: Binding failed for port c5eddc15-1383-4b48-be39-d5589ce77368, please check neutron logs for more information. [ 652.924633] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] [ 652.924633] env[63593]: INFO nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Terminating instance [ 652.926795] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "refresh_cache-81b0b694-b9fb-4416-8751-7f1e24e96869" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 652.927028] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquired lock "refresh_cache-81b0b694-b9fb-4416-8751-7f1e24e96869" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 652.927552] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 652.993436] env[63593]: DEBUG nova.compute.manager [req-b9e6de2e-fe65-41e3-875f-c43ad6e005be req-6fd9ec37-5b2e-45e6-9730-64274f66b32f service nova] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Received event network-changed-c5eddc15-1383-4b48-be39-d5589ce77368 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 652.993632] env[63593]: DEBUG nova.compute.manager [req-b9e6de2e-fe65-41e3-875f-c43ad6e005be req-6fd9ec37-5b2e-45e6-9730-64274f66b32f service nova] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Refreshing instance network info cache due to event network-changed-c5eddc15-1383-4b48-be39-d5589ce77368. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 652.993865] env[63593]: DEBUG oslo_concurrency.lockutils [req-b9e6de2e-fe65-41e3-875f-c43ad6e005be req-6fd9ec37-5b2e-45e6-9730-64274f66b32f service nova] Acquiring lock "refresh_cache-81b0b694-b9fb-4416-8751-7f1e24e96869" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 653.248160] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74dfd0b5-ee4b-4b40-8b7d-54025a566c44 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.256921] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc42b698-b805-4458-8d1a-6bc8f00d777a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.294407] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3aefcdaa-ad00-4dcc-90d8-02496aa0f9a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.303764] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97be2ac9-d284-44d1-b442-6a6605c03c17 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.321849] env[63593]: DEBUG nova.compute.provider_tree [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 653.420785] env[63593]: WARNING oslo_vmware.rw_handles [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 653.420785] env[63593]: ERROR oslo_vmware.rw_handles [ 653.421197] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/7ecb1281-97e2-44cc-afcf-db640b2c903d/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 653.423317] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 653.423620] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Copying Virtual Disk [datastore2] vmware_temp/7ecb1281-97e2-44cc-afcf-db640b2c903d/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/7ecb1281-97e2-44cc-afcf-db640b2c903d/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 653.423976] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-c0bf7da1-3d3f-426d-8863-30b6fb884244 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.436261] env[63593]: DEBUG oslo_vmware.api [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Waiting for the task: (returnval){ [ 653.436261] env[63593]: value = "task-1367947" [ 653.436261] env[63593]: _type = "Task" [ 653.436261] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 653.446368] env[63593]: DEBUG oslo_vmware.api [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Task: {'id': task-1367947, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 653.456106] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 653.657758] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 653.828097] env[63593]: DEBUG nova.scheduler.client.report [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 653.870655] env[63593]: DEBUG nova.compute.manager [req-4c3e186c-0337-4a97-962f-ca691e026123 req-8c8f8a23-ea49-41d7-964d-fefe72ecff44 service nova] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Received event network-vif-deleted-34edec86-cf21-4cd5-8dfd-219a396ba46b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 653.898944] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 653.925200] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:54:04Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='839156237',id=24,is_public=True,memory_mb=192,name='tempest-flavor_with_ephemeral_0-1187888855',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 653.925941] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 653.926253] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 653.926637] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 653.926932] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 653.927435] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 653.927756] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 653.928050] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 653.928332] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 653.928616] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 653.928896] env[63593]: DEBUG nova.virt.hardware [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 653.930016] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-214858cc-4460-45ae-a6f1-81c802fef685 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.944432] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9a9db34-e2af-4d9f-9312-7335d756704e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.954287] env[63593]: DEBUG oslo_vmware.exceptions [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 653.965291] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 653.965291] env[63593]: ERROR nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 653.965291] env[63593]: Faults: ['InvalidArgument'] [ 653.965291] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Traceback (most recent call last): [ 653.965291] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 653.965291] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] yield resources [ 653.965291] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 653.965291] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] self.driver.spawn(context, instance, image_meta, [ 653.965291] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 653.965291] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] self._vmops.spawn(context, instance, image_meta, injected_files, [ 653.965291] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] self._fetch_image_if_missing(context, vi) [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] image_cache(vi, tmp_image_ds_loc) [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] vm_util.copy_virtual_disk( [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] session._wait_for_task(vmdk_copy_task) [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] return self.wait_for_task(task_ref) [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] return evt.wait() [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 653.965660] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] result = hub.switch() [ 653.966042] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 653.966042] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] return self.greenlet.switch() [ 653.966042] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 653.966042] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] self.f(*self.args, **self.kw) [ 653.966042] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 653.966042] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] raise exceptions.translate_fault(task_info.error) [ 653.966042] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 653.966042] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Faults: ['InvalidArgument'] [ 653.966042] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] [ 653.966042] env[63593]: INFO nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Terminating instance [ 653.966979] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 653.967212] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 653.967430] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-97b96107-8bed-4982-8e45-04700d0932bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.970157] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquiring lock "refresh_cache-7044b799-74ed-4711-9b53-08f8a8ddbb78" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 653.970157] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquired lock "refresh_cache-7044b799-74ed-4711-9b53-08f8a8ddbb78" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 653.970298] env[63593]: DEBUG nova.network.neutron [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 653.978673] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 653.978850] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 653.979572] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-909f6150-5522-41af-8a5b-fa49f1228364 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.986387] env[63593]: DEBUG oslo_vmware.api [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Waiting for the task: (returnval){ [ 653.986387] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5255345b-1ba2-22a2-428d-18375f1ea43f" [ 653.986387] env[63593]: _type = "Task" [ 653.986387] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 653.999022] env[63593]: DEBUG oslo_vmware.api [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5255345b-1ba2-22a2-428d-18375f1ea43f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 654.162233] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Releasing lock "refresh_cache-81b0b694-b9fb-4416-8751-7f1e24e96869" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 654.162233] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 654.162233] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 654.162233] env[63593]: DEBUG oslo_concurrency.lockutils [req-b9e6de2e-fe65-41e3-875f-c43ad6e005be req-6fd9ec37-5b2e-45e6-9730-64274f66b32f service nova] Acquired lock "refresh_cache-81b0b694-b9fb-4416-8751-7f1e24e96869" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 654.162889] env[63593]: DEBUG nova.network.neutron [req-b9e6de2e-fe65-41e3-875f-c43ad6e005be req-6fd9ec37-5b2e-45e6-9730-64274f66b32f service nova] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Refreshing network info cache for port c5eddc15-1383-4b48-be39-d5589ce77368 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 654.164551] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-33a423d8-4e06-4b48-87cf-683f572a352b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.177786] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-034ad4e5-60b5-4b7a-b206-80ce2c813c0d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.208629] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 81b0b694-b9fb-4416-8751-7f1e24e96869 could not be found. [ 654.208856] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 654.209059] env[63593]: INFO nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Took 0.05 seconds to destroy the instance on the hypervisor. [ 654.209307] env[63593]: DEBUG oslo.service.loopingcall [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 654.209779] env[63593]: DEBUG nova.compute.manager [-] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 654.209931] env[63593]: DEBUG nova.network.neutron [-] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 654.233086] env[63593]: DEBUG nova.network.neutron [-] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 654.336341] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.458s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 654.336341] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 654.339592] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 9.994s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 654.494401] env[63593]: DEBUG nova.network.neutron [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 654.498201] env[63593]: ERROR nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 289407e8-c816-4ada-a146-53327b567f6c, please check neutron logs for more information. [ 654.498201] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 654.498201] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.498201] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 654.498201] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 654.498201] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 654.498201] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 654.498201] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 654.498201] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.498201] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 654.498201] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.498201] env[63593]: ERROR nova.compute.manager raise self.value [ 654.498201] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 654.498201] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 654.498201] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.498201] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 654.498743] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.498743] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 654.498743] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 289407e8-c816-4ada-a146-53327b567f6c, please check neutron logs for more information. [ 654.498743] env[63593]: ERROR nova.compute.manager [ 654.498743] env[63593]: Traceback (most recent call last): [ 654.498743] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 654.498743] env[63593]: listener.cb(fileno) [ 654.498743] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 654.498743] env[63593]: result = function(*args, **kwargs) [ 654.498743] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 654.498743] env[63593]: return func(*args, **kwargs) [ 654.498743] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 654.498743] env[63593]: raise e [ 654.498743] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.498743] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 654.498743] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 654.498743] env[63593]: created_port_ids = self._update_ports_for_instance( [ 654.498743] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 654.498743] env[63593]: with excutils.save_and_reraise_exception(): [ 654.498743] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.498743] env[63593]: self.force_reraise() [ 654.498743] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.498743] env[63593]: raise self.value [ 654.498743] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 654.498743] env[63593]: updated_port = self._update_port( [ 654.498743] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.498743] env[63593]: _ensure_no_port_binding_failure(port) [ 654.498743] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.498743] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 654.499574] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 289407e8-c816-4ada-a146-53327b567f6c, please check neutron logs for more information. [ 654.499574] env[63593]: Removing descriptor: 19 [ 654.504953] env[63593]: ERROR nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 289407e8-c816-4ada-a146-53327b567f6c, please check neutron logs for more information. [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Traceback (most recent call last): [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] yield resources [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self.driver.spawn(context, instance, image_meta, [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] vm_ref = self.build_virtual_machine(instance, [ 654.504953] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] vif_infos = vmwarevif.get_vif_info(self._session, [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] for vif in network_info: [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] return self._sync_wrapper(fn, *args, **kwargs) [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self.wait() [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self[:] = self._gt.wait() [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] return self._exit_event.wait() [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 654.505508] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] result = hub.switch() [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] return self.greenlet.switch() [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] result = function(*args, **kwargs) [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] return func(*args, **kwargs) [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] raise e [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] nwinfo = self.network_api.allocate_for_instance( [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] created_port_ids = self._update_ports_for_instance( [ 654.506114] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] with excutils.save_and_reraise_exception(): [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self.force_reraise() [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] raise self.value [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] updated_port = self._update_port( [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] _ensure_no_port_binding_failure(port) [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] raise exception.PortBindingFailed(port_id=port['id']) [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] nova.exception.PortBindingFailed: Binding failed for port 289407e8-c816-4ada-a146-53327b567f6c, please check neutron logs for more information. [ 654.506711] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] [ 654.507243] env[63593]: INFO nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Terminating instance [ 654.507243] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 654.507243] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Creating directory with path [datastore2] vmware_temp/9c36615a-8b1a-4107-93e0-63eb7b1c5e27/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 654.507381] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f0f09838-2da1-47d4-8208-43835575f6dd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.510477] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Acquiring lock "refresh_cache-c6d88cbe-488b-490b-a37c-51a311276dfe" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.510654] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Acquired lock "refresh_cache-c6d88cbe-488b-490b-a37c-51a311276dfe" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 654.510847] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 654.523673] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Created directory with path [datastore2] vmware_temp/9c36615a-8b1a-4107-93e0-63eb7b1c5e27/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 654.523915] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Fetch image to [datastore2] vmware_temp/9c36615a-8b1a-4107-93e0-63eb7b1c5e27/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 654.524102] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/9c36615a-8b1a-4107-93e0-63eb7b1c5e27/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 654.526904] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e61891a8-3f81-45b9-95bd-68d42ad95f40 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.535569] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b45aa6c8-2932-4a09-a079-3327d4bdacce {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.547971] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35699c24-6cd8-4fe1-801f-226acb565caf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.586224] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe26bf29-ebfc-4884-9b3d-5f08ff69cbbc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.592966] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4ecae99c-4940-4392-b3ea-e1882dd01652 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 654.664872] env[63593]: DEBUG nova.network.neutron [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 654.685866] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 654.696951] env[63593]: DEBUG nova.network.neutron [req-b9e6de2e-fe65-41e3-875f-c43ad6e005be req-6fd9ec37-5b2e-45e6-9730-64274f66b32f service nova] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 654.736238] env[63593]: DEBUG nova.network.neutron [-] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 654.751628] env[63593]: DEBUG oslo_vmware.rw_handles [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/9c36615a-8b1a-4107-93e0-63eb7b1c5e27/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 654.818386] env[63593]: DEBUG oslo_vmware.rw_handles [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 654.818386] env[63593]: DEBUG oslo_vmware.rw_handles [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/9c36615a-8b1a-4107-93e0-63eb7b1c5e27/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 654.848888] env[63593]: DEBUG nova.compute.utils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 654.854682] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 654.854793] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 654.876651] env[63593]: DEBUG nova.network.neutron [req-b9e6de2e-fe65-41e3-875f-c43ad6e005be req-6fd9ec37-5b2e-45e6-9730-64274f66b32f service nova] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 654.942029] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "70595430-6ab6-49d4-b8fe-131bd44de838" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 654.943585] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "70595430-6ab6-49d4-b8fe-131bd44de838" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.002s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 654.963578] env[63593]: DEBUG nova.policy [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cf0cd9ae44ff44d4970d0a22019ef595', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ef85f19f990f4020a128a81440ab1e6f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 655.033516] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.170619] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Releasing lock "refresh_cache-7044b799-74ed-4711-9b53-08f8a8ddbb78" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.172015] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 655.172274] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 655.173788] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de14d904-e9df-4cd5-87b0-e0a3a0b97f3d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.179468] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.190151] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 655.190151] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-952228a9-f041-48b6-8f8b-f55ced8bc619 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.230928] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 655.231065] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 655.231224] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Deleting the datastore file [datastore2] 7044b799-74ed-4711-9b53-08f8a8ddbb78 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 655.232493] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-3ca2f45a-a408-45d8-a8bd-8585dcc83bf4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.238493] env[63593]: INFO nova.compute.manager [-] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Took 1.03 seconds to deallocate network for instance. [ 655.245298] env[63593]: DEBUG oslo_vmware.api [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Waiting for the task: (returnval){ [ 655.245298] env[63593]: value = "task-1367950" [ 655.245298] env[63593]: _type = "Task" [ 655.245298] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 655.245506] env[63593]: DEBUG nova.compute.claims [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 655.245678] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 655.259962] env[63593]: DEBUG oslo_vmware.api [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Task: {'id': task-1367950, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 655.301662] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb4030a5-5d3b-4d93-a95e-14c7378d4fbe {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.309715] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6d9079a-0d3f-4685-8b8e-13c89c35dc03 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.344711] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8266f000-b9da-46f8-8ff0-2d8f4a7b0785 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.353241] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d80a44b-76e0-4ca5-8dfe-94d6b9908601 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.357870] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 655.371509] env[63593]: DEBUG nova.compute.provider_tree [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 655.382488] env[63593]: DEBUG oslo_concurrency.lockutils [req-b9e6de2e-fe65-41e3-875f-c43ad6e005be req-6fd9ec37-5b2e-45e6-9730-64274f66b32f service nova] Releasing lock "refresh_cache-81b0b694-b9fb-4416-8751-7f1e24e96869" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.658031] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Successfully created port: bcb30ba2-9252-4597-bc57-0c9e2853ec60 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 655.686229] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Releasing lock "refresh_cache-c6d88cbe-488b-490b-a37c-51a311276dfe" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.686229] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 655.686229] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 655.686229] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7a9bc787-3ac4-4ae3-8a80-bb86168c75b1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.696232] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b93cc5da-1a2e-418c-9fc4-34ba719b2353 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.729787] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c6d88cbe-488b-490b-a37c-51a311276dfe could not be found. [ 655.730768] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 655.730768] env[63593]: INFO nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Took 0.05 seconds to destroy the instance on the hypervisor. [ 655.730768] env[63593]: DEBUG oslo.service.loopingcall [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 655.730950] env[63593]: DEBUG nova.compute.manager [-] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 655.731355] env[63593]: DEBUG nova.network.neutron [-] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 655.756207] env[63593]: DEBUG nova.network.neutron [-] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.761082] env[63593]: DEBUG oslo_vmware.api [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Task: {'id': task-1367950, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.051847} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 655.761438] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 655.761675] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 655.761795] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 655.761955] env[63593]: INFO nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Took 0.59 seconds to destroy the instance on the hypervisor. [ 655.762202] env[63593]: DEBUG oslo.service.loopingcall [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 655.762409] env[63593]: DEBUG nova.compute.manager [-] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 655.765031] env[63593]: DEBUG nova.compute.claims [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 655.765031] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 655.879799] env[63593]: DEBUG nova.scheduler.client.report [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 656.263794] env[63593]: DEBUG nova.network.neutron [-] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 656.369278] env[63593]: DEBUG nova.compute.manager [req-ee1f6a40-02c1-45ac-85ee-820a85bcae59 req-12ae41d9-902e-43bb-88d8-e0fe0b65cee7 service nova] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Received event network-vif-deleted-c5eddc15-1383-4b48-be39-d5589ce77368 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 656.370326] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 656.388514] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.046s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.388514] env[63593]: ERROR nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 964fa279-e5ea-4fa0-b942-120129c86d59, please check neutron logs for more information. [ 656.388514] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Traceback (most recent call last): [ 656.388514] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 656.388514] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self.driver.spawn(context, instance, image_meta, [ 656.388514] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 656.388514] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self._vmops.spawn(context, instance, image_meta, injected_files, [ 656.388514] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 656.388514] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] vm_ref = self.build_virtual_machine(instance, [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] vif_infos = vmwarevif.get_vif_info(self._session, [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] for vif in network_info: [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] return self._sync_wrapper(fn, *args, **kwargs) [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self.wait() [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self[:] = self._gt.wait() [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] return self._exit_event.wait() [ 656.388965] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] result = hub.switch() [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] return self.greenlet.switch() [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] result = function(*args, **kwargs) [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] return func(*args, **kwargs) [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] raise e [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] nwinfo = self.network_api.allocate_for_instance( [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 656.389643] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] created_port_ids = self._update_ports_for_instance( [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] with excutils.save_and_reraise_exception(): [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] self.force_reraise() [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] raise self.value [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] updated_port = self._update_port( [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] _ensure_no_port_binding_failure(port) [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 656.390043] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] raise exception.PortBindingFailed(port_id=port['id']) [ 656.390365] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] nova.exception.PortBindingFailed: Binding failed for port 964fa279-e5ea-4fa0-b942-120129c86d59, please check neutron logs for more information. [ 656.390365] env[63593]: ERROR nova.compute.manager [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] [ 656.390365] env[63593]: DEBUG nova.compute.utils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Binding failed for port 964fa279-e5ea-4fa0-b942-120129c86d59, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 656.393431] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.308s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 656.393431] env[63593]: INFO nova.compute.claims [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 656.395576] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Build of instance 43bcc8c6-0fd5-467b-9e11-3b788cf04236 was re-scheduled: Binding failed for port 964fa279-e5ea-4fa0-b942-120129c86d59, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 656.396088] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 656.400138] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Acquiring lock "refresh_cache-43bcc8c6-0fd5-467b-9e11-3b788cf04236" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 656.400323] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Acquired lock "refresh_cache-43bcc8c6-0fd5-467b-9e11-3b788cf04236" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 656.400488] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 656.414403] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 656.414644] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 656.414816] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 656.415105] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 656.415275] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 656.415421] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 656.415627] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 656.415848] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 656.415988] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 656.416138] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 656.416304] env[63593]: DEBUG nova.virt.hardware [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 656.417384] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05f0a6db-1216-4099-b149-39eb5e4d6318 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 656.425896] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42402535-e1ff-4df6-892a-de7d15e9fa73 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 656.765185] env[63593]: INFO nova.compute.manager [-] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Took 1.03 seconds to deallocate network for instance. [ 656.770173] env[63593]: DEBUG nova.compute.claims [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 656.773753] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.943665] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 657.050490] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 657.140131] env[63593]: DEBUG nova.compute.manager [req-83e39d74-9a84-46df-9384-472a658bb142 req-2dbc2efa-7c84-4e24-828c-4c45080aca48 service nova] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Received event network-changed-289407e8-c816-4ada-a146-53327b567f6c {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 657.140131] env[63593]: DEBUG nova.compute.manager [req-83e39d74-9a84-46df-9384-472a658bb142 req-2dbc2efa-7c84-4e24-828c-4c45080aca48 service nova] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Refreshing instance network info cache due to event network-changed-289407e8-c816-4ada-a146-53327b567f6c. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 657.140131] env[63593]: DEBUG oslo_concurrency.lockutils [req-83e39d74-9a84-46df-9384-472a658bb142 req-2dbc2efa-7c84-4e24-828c-4c45080aca48 service nova] Acquiring lock "refresh_cache-c6d88cbe-488b-490b-a37c-51a311276dfe" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 657.142113] env[63593]: DEBUG oslo_concurrency.lockutils [req-83e39d74-9a84-46df-9384-472a658bb142 req-2dbc2efa-7c84-4e24-828c-4c45080aca48 service nova] Acquired lock "refresh_cache-c6d88cbe-488b-490b-a37c-51a311276dfe" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 657.143039] env[63593]: DEBUG nova.network.neutron [req-83e39d74-9a84-46df-9384-472a658bb142 req-2dbc2efa-7c84-4e24-828c-4c45080aca48 service nova] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Refreshing network info cache for port 289407e8-c816-4ada-a146-53327b567f6c {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 657.181050] env[63593]: ERROR nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port bcb30ba2-9252-4597-bc57-0c9e2853ec60, please check neutron logs for more information. [ 657.181050] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 657.181050] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 657.181050] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 657.181050] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 657.181050] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 657.181050] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 657.181050] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 657.181050] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 657.181050] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 657.181050] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 657.181050] env[63593]: ERROR nova.compute.manager raise self.value [ 657.181050] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 657.181050] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 657.181050] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 657.181050] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 657.182939] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 657.182939] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 657.182939] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port bcb30ba2-9252-4597-bc57-0c9e2853ec60, please check neutron logs for more information. [ 657.182939] env[63593]: ERROR nova.compute.manager [ 657.182939] env[63593]: Traceback (most recent call last): [ 657.182939] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 657.182939] env[63593]: listener.cb(fileno) [ 657.182939] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 657.182939] env[63593]: result = function(*args, **kwargs) [ 657.182939] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 657.182939] env[63593]: return func(*args, **kwargs) [ 657.182939] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 657.182939] env[63593]: raise e [ 657.182939] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 657.182939] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 657.182939] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 657.182939] env[63593]: created_port_ids = self._update_ports_for_instance( [ 657.182939] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 657.182939] env[63593]: with excutils.save_and_reraise_exception(): [ 657.182939] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 657.182939] env[63593]: self.force_reraise() [ 657.182939] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 657.182939] env[63593]: raise self.value [ 657.182939] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 657.182939] env[63593]: updated_port = self._update_port( [ 657.182939] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 657.182939] env[63593]: _ensure_no_port_binding_failure(port) [ 657.182939] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 657.182939] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 657.184418] env[63593]: nova.exception.PortBindingFailed: Binding failed for port bcb30ba2-9252-4597-bc57-0c9e2853ec60, please check neutron logs for more information. [ 657.184418] env[63593]: Removing descriptor: 19 [ 657.184418] env[63593]: ERROR nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port bcb30ba2-9252-4597-bc57-0c9e2853ec60, please check neutron logs for more information. [ 657.184418] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Traceback (most recent call last): [ 657.184418] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 657.184418] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] yield resources [ 657.184418] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 657.184418] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self.driver.spawn(context, instance, image_meta, [ 657.184418] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 657.184418] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 657.184418] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 657.184418] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] vm_ref = self.build_virtual_machine(instance, [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] vif_infos = vmwarevif.get_vif_info(self._session, [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] for vif in network_info: [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] return self._sync_wrapper(fn, *args, **kwargs) [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self.wait() [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self[:] = self._gt.wait() [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] return self._exit_event.wait() [ 657.185128] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] result = hub.switch() [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] return self.greenlet.switch() [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] result = function(*args, **kwargs) [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] return func(*args, **kwargs) [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] raise e [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] nwinfo = self.network_api.allocate_for_instance( [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 657.185785] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] created_port_ids = self._update_ports_for_instance( [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] with excutils.save_and_reraise_exception(): [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self.force_reraise() [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] raise self.value [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] updated_port = self._update_port( [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] _ensure_no_port_binding_failure(port) [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 657.186412] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] raise exception.PortBindingFailed(port_id=port['id']) [ 657.186867] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] nova.exception.PortBindingFailed: Binding failed for port bcb30ba2-9252-4597-bc57-0c9e2853ec60, please check neutron logs for more information. [ 657.186867] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] [ 657.186867] env[63593]: INFO nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Terminating instance [ 657.186867] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Acquiring lock "refresh_cache-6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 657.186867] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Acquired lock "refresh_cache-6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 657.186867] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 657.459350] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "006d2ab8-e38b-4b69-b3b2-115c39e44218" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 657.459632] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "006d2ab8-e38b-4b69-b3b2-115c39e44218" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 657.551580] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Releasing lock "refresh_cache-43bcc8c6-0fd5-467b-9e11-3b788cf04236" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 657.552559] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 657.552559] env[63593]: DEBUG nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 657.552559] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 657.575200] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 657.682637] env[63593]: DEBUG nova.network.neutron [req-83e39d74-9a84-46df-9384-472a658bb142 req-2dbc2efa-7c84-4e24-828c-4c45080aca48 service nova] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 657.725908] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 657.807051] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 657.834722] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-216fac8c-ebc5-43d1-b851-0495438908f8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.841844] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a75904c9-ed5c-4c33-80c8-4098ce260164 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.876802] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ac9a51a-a06b-43b1-af08-08b19563dbf4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.884642] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cb03584-e566-4194-a70e-29594306762d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.901183] env[63593]: DEBUG nova.compute.provider_tree [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 657.903115] env[63593]: DEBUG nova.network.neutron [req-83e39d74-9a84-46df-9384-472a658bb142 req-2dbc2efa-7c84-4e24-828c-4c45080aca48 service nova] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 658.083033] env[63593]: DEBUG nova.network.neutron [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 658.312298] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Releasing lock "refresh_cache-6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 658.312298] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 658.312298] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 658.312298] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-64d4b22e-7f18-4101-8688-d2d57a29bcc1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.321370] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5433fd7b-f845-41ab-a7df-f8e4e4af4be2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.349531] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1 could not be found. [ 658.349766] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 658.349936] env[63593]: INFO nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 658.350204] env[63593]: DEBUG oslo.service.loopingcall [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 658.350843] env[63593]: DEBUG nova.compute.manager [-] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 658.350843] env[63593]: DEBUG nova.network.neutron [-] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 658.384024] env[63593]: DEBUG nova.network.neutron [-] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 658.406028] env[63593]: DEBUG nova.scheduler.client.report [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 658.411994] env[63593]: DEBUG oslo_concurrency.lockutils [req-83e39d74-9a84-46df-9384-472a658bb142 req-2dbc2efa-7c84-4e24-828c-4c45080aca48 service nova] Releasing lock "refresh_cache-c6d88cbe-488b-490b-a37c-51a311276dfe" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 658.412711] env[63593]: DEBUG nova.compute.manager [req-83e39d74-9a84-46df-9384-472a658bb142 req-2dbc2efa-7c84-4e24-828c-4c45080aca48 service nova] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Received event network-vif-deleted-289407e8-c816-4ada-a146-53327b567f6c {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 658.586299] env[63593]: INFO nova.compute.manager [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] [instance: 43bcc8c6-0fd5-467b-9e11-3b788cf04236] Took 1.03 seconds to deallocate network for instance. [ 658.885596] env[63593]: DEBUG nova.network.neutron [-] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 658.916380] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.527s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 658.916380] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 658.918322] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.815s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 658.921036] env[63593]: INFO nova.compute.claims [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 659.149953] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Acquiring lock "eddf892e-9706-4d33-8241-f8df6c29dee4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 659.150368] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Lock "eddf892e-9706-4d33-8241-f8df6c29dee4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 659.389300] env[63593]: INFO nova.compute.manager [-] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Took 1.04 seconds to deallocate network for instance. [ 659.392762] env[63593]: DEBUG nova.compute.claims [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 659.393067] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 659.420214] env[63593]: DEBUG nova.compute.utils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 659.421679] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 659.421858] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 659.482321] env[63593]: DEBUG nova.policy [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd5f23913d4a1411282f081fb5ed46a11', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '220c072768c1441fbfe88f849cd6eb03', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 659.634996] env[63593]: INFO nova.scheduler.client.report [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Deleted allocations for instance 43bcc8c6-0fd5-467b-9e11-3b788cf04236 [ 659.712483] env[63593]: DEBUG nova.compute.manager [req-0fd6f918-fdc2-4f82-9d2f-1dfa04215ede req-855a5946-c271-4511-a100-bd827174699b service nova] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Received event network-changed-bcb30ba2-9252-4597-bc57-0c9e2853ec60 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 659.712786] env[63593]: DEBUG nova.compute.manager [req-0fd6f918-fdc2-4f82-9d2f-1dfa04215ede req-855a5946-c271-4511-a100-bd827174699b service nova] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Refreshing instance network info cache due to event network-changed-bcb30ba2-9252-4597-bc57-0c9e2853ec60. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 659.713091] env[63593]: DEBUG oslo_concurrency.lockutils [req-0fd6f918-fdc2-4f82-9d2f-1dfa04215ede req-855a5946-c271-4511-a100-bd827174699b service nova] Acquiring lock "refresh_cache-6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 659.713341] env[63593]: DEBUG oslo_concurrency.lockutils [req-0fd6f918-fdc2-4f82-9d2f-1dfa04215ede req-855a5946-c271-4511-a100-bd827174699b service nova] Acquired lock "refresh_cache-6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 659.713470] env[63593]: DEBUG nova.network.neutron [req-0fd6f918-fdc2-4f82-9d2f-1dfa04215ede req-855a5946-c271-4511-a100-bd827174699b service nova] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Refreshing network info cache for port bcb30ba2-9252-4597-bc57-0c9e2853ec60 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 659.927576] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 660.090363] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Successfully created port: cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 660.146825] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d193a386-bd4a-4026-9a0d-78b63c0c36ca tempest-ServerDiagnosticsNegativeTest-1482363254 tempest-ServerDiagnosticsNegativeTest-1482363254-project-member] Lock "43bcc8c6-0fd5-467b-9e11-3b788cf04236" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 48.599s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 660.155242] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Acquiring lock "c99cbfae-db59-4b84-b8ab-09b84ebe4623" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 660.155459] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Lock "c99cbfae-db59-4b84-b8ab-09b84ebe4623" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 660.253961] env[63593]: DEBUG nova.network.neutron [req-0fd6f918-fdc2-4f82-9d2f-1dfa04215ede req-855a5946-c271-4511-a100-bd827174699b service nova] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 660.371456] env[63593]: DEBUG nova.network.neutron [req-0fd6f918-fdc2-4f82-9d2f-1dfa04215ede req-855a5946-c271-4511-a100-bd827174699b service nova] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 660.383726] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c3610a5-40f2-4188-84da-5b99ffb27947 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.392894] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08f9790e-bcd7-4d1b-aa68-c30caf8073b6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.431131] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d79db42b-f34d-4e31-8ebf-51436e065747 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.439028] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-270e3211-2a49-42e9-b257-cf19f7bf4a66 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.455079] env[63593]: DEBUG nova.compute.provider_tree [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 660.650354] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 660.877220] env[63593]: DEBUG oslo_concurrency.lockutils [req-0fd6f918-fdc2-4f82-9d2f-1dfa04215ede req-855a5946-c271-4511-a100-bd827174699b service nova] Releasing lock "refresh_cache-6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 660.877503] env[63593]: DEBUG nova.compute.manager [req-0fd6f918-fdc2-4f82-9d2f-1dfa04215ede req-855a5946-c271-4511-a100-bd827174699b service nova] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Received event network-vif-deleted-bcb30ba2-9252-4597-bc57-0c9e2853ec60 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 660.951281] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 660.960674] env[63593]: DEBUG nova.scheduler.client.report [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 660.981923] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 660.982177] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 660.982331] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 660.982508] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 660.982666] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 660.982791] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 660.982997] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 660.983773] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 660.983773] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 660.983856] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 660.985498] env[63593]: DEBUG nova.virt.hardware [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 660.985498] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d64ef872-da7e-4386-ab09-f4dd6505b7f7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.997625] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b877d8f2-5e14-4186-9fd0-e97ce1245eca {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 661.187573] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 661.472372] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.553s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 661.472372] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 661.475733] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.258s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 661.477178] env[63593]: INFO nova.compute.claims [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 661.993967] env[63593]: DEBUG nova.compute.utils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 661.998444] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 661.998617] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 662.071511] env[63593]: DEBUG nova.compute.manager [req-55b98dc2-c394-4589-a72a-d68ef80c8a30 req-6c3fdb8a-a53e-44b8-9aa3-aa0990727aa7 service nova] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Received event network-changed-cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 662.071705] env[63593]: DEBUG nova.compute.manager [req-55b98dc2-c394-4589-a72a-d68ef80c8a30 req-6c3fdb8a-a53e-44b8-9aa3-aa0990727aa7 service nova] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Refreshing instance network info cache due to event network-changed-cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 662.071919] env[63593]: DEBUG oslo_concurrency.lockutils [req-55b98dc2-c394-4589-a72a-d68ef80c8a30 req-6c3fdb8a-a53e-44b8-9aa3-aa0990727aa7 service nova] Acquiring lock "refresh_cache-bb3a0a79-ef74-4fb0-a286-91b1b92df751" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 662.072068] env[63593]: DEBUG oslo_concurrency.lockutils [req-55b98dc2-c394-4589-a72a-d68ef80c8a30 req-6c3fdb8a-a53e-44b8-9aa3-aa0990727aa7 service nova] Acquired lock "refresh_cache-bb3a0a79-ef74-4fb0-a286-91b1b92df751" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 662.072569] env[63593]: DEBUG nova.network.neutron [req-55b98dc2-c394-4589-a72a-d68ef80c8a30 req-6c3fdb8a-a53e-44b8-9aa3-aa0990727aa7 service nova] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Refreshing network info cache for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 662.103946] env[63593]: DEBUG nova.policy [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a7d245a2cfdc4d2c8f56ee3092b47150', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5f9ef94e1ee84dceaa0da70d94a13020', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 662.198744] env[63593]: ERROR nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c, please check neutron logs for more information. [ 662.198744] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 662.198744] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 662.198744] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 662.198744] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 662.198744] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 662.198744] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 662.198744] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 662.198744] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 662.198744] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 662.198744] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 662.198744] env[63593]: ERROR nova.compute.manager raise self.value [ 662.198744] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 662.198744] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 662.198744] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 662.198744] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 662.199443] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 662.199443] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 662.199443] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c, please check neutron logs for more information. [ 662.199443] env[63593]: ERROR nova.compute.manager [ 662.199443] env[63593]: Traceback (most recent call last): [ 662.199443] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 662.199443] env[63593]: listener.cb(fileno) [ 662.199443] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 662.199443] env[63593]: result = function(*args, **kwargs) [ 662.199443] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 662.199443] env[63593]: return func(*args, **kwargs) [ 662.199443] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 662.199443] env[63593]: raise e [ 662.199443] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 662.199443] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 662.199443] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 662.199443] env[63593]: created_port_ids = self._update_ports_for_instance( [ 662.199443] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 662.199443] env[63593]: with excutils.save_and_reraise_exception(): [ 662.199443] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 662.199443] env[63593]: self.force_reraise() [ 662.199443] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 662.199443] env[63593]: raise self.value [ 662.199443] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 662.199443] env[63593]: updated_port = self._update_port( [ 662.199443] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 662.199443] env[63593]: _ensure_no_port_binding_failure(port) [ 662.199443] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 662.199443] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 662.200384] env[63593]: nova.exception.PortBindingFailed: Binding failed for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c, please check neutron logs for more information. [ 662.200384] env[63593]: Removing descriptor: 19 [ 662.200384] env[63593]: ERROR nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c, please check neutron logs for more information. [ 662.200384] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Traceback (most recent call last): [ 662.200384] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 662.200384] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] yield resources [ 662.200384] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 662.200384] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self.driver.spawn(context, instance, image_meta, [ 662.200384] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 662.200384] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self._vmops.spawn(context, instance, image_meta, injected_files, [ 662.200384] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 662.200384] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] vm_ref = self.build_virtual_machine(instance, [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] vif_infos = vmwarevif.get_vif_info(self._session, [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] for vif in network_info: [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] return self._sync_wrapper(fn, *args, **kwargs) [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self.wait() [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self[:] = self._gt.wait() [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] return self._exit_event.wait() [ 662.200745] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] result = hub.switch() [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] return self.greenlet.switch() [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] result = function(*args, **kwargs) [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] return func(*args, **kwargs) [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] raise e [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] nwinfo = self.network_api.allocate_for_instance( [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 662.201156] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] created_port_ids = self._update_ports_for_instance( [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] with excutils.save_and_reraise_exception(): [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self.force_reraise() [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] raise self.value [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] updated_port = self._update_port( [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] _ensure_no_port_binding_failure(port) [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 662.201521] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] raise exception.PortBindingFailed(port_id=port['id']) [ 662.201891] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] nova.exception.PortBindingFailed: Binding failed for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c, please check neutron logs for more information. [ 662.201891] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] [ 662.201891] env[63593]: INFO nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Terminating instance [ 662.204199] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "refresh_cache-bb3a0a79-ef74-4fb0-a286-91b1b92df751" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 662.501771] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 662.616031] env[63593]: DEBUG nova.network.neutron [req-55b98dc2-c394-4589-a72a-d68ef80c8a30 req-6c3fdb8a-a53e-44b8-9aa3-aa0990727aa7 service nova] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 662.863820] env[63593]: DEBUG nova.network.neutron [req-55b98dc2-c394-4589-a72a-d68ef80c8a30 req-6c3fdb8a-a53e-44b8-9aa3-aa0990727aa7 service nova] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 662.911032] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e5f0f35-bf80-41f7-a4f2-35a06ac1e535 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.919149] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48463463-67f9-46d4-a592-d7cd2590db3c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.960616] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85b260e6-d29f-4295-bc72-5f15ec3ce4d5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.967655] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec049d79-016f-4ca0-b37b-d534421dfc42 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.982147] env[63593]: DEBUG nova.compute.provider_tree [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 663.056205] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Successfully created port: 0192162f-7b9c-4e13-9f0e-51910b514bed {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 663.366637] env[63593]: DEBUG oslo_concurrency.lockutils [req-55b98dc2-c394-4589-a72a-d68ef80c8a30 req-6c3fdb8a-a53e-44b8-9aa3-aa0990727aa7 service nova] Releasing lock "refresh_cache-bb3a0a79-ef74-4fb0-a286-91b1b92df751" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 663.367192] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "refresh_cache-bb3a0a79-ef74-4fb0-a286-91b1b92df751" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 663.367379] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 663.489340] env[63593]: DEBUG nova.scheduler.client.report [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 663.512408] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 663.543634] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 663.543876] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 663.544140] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 663.544356] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 663.544356] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 663.544494] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 663.544692] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 663.544920] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 663.545014] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 663.545174] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 663.545344] env[63593]: DEBUG nova.virt.hardware [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 663.546748] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e550708-6ed8-43d0-94b0-30fc2cde9938 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.558784] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dda1d8ae-b54c-4164-bba1-0b307f145c18 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 663.911815] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 663.996077] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.518s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 663.996077] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 663.998382] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.360s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.200520] env[63593]: DEBUG nova.compute.manager [req-423ba74e-a121-4254-8f9f-890993843b88 req-ef4f9218-f60f-4f7f-b5c3-753949f9971a service nova] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Received event network-changed-0192162f-7b9c-4e13-9f0e-51910b514bed {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 664.200761] env[63593]: DEBUG nova.compute.manager [req-423ba74e-a121-4254-8f9f-890993843b88 req-ef4f9218-f60f-4f7f-b5c3-753949f9971a service nova] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Refreshing instance network info cache due to event network-changed-0192162f-7b9c-4e13-9f0e-51910b514bed. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 664.200879] env[63593]: DEBUG oslo_concurrency.lockutils [req-423ba74e-a121-4254-8f9f-890993843b88 req-ef4f9218-f60f-4f7f-b5c3-753949f9971a service nova] Acquiring lock "refresh_cache-4869e3e4-48bb-446f-a15e-3a53bae1bc9b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 664.201070] env[63593]: DEBUG oslo_concurrency.lockutils [req-423ba74e-a121-4254-8f9f-890993843b88 req-ef4f9218-f60f-4f7f-b5c3-753949f9971a service nova] Acquired lock "refresh_cache-4869e3e4-48bb-446f-a15e-3a53bae1bc9b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 664.201384] env[63593]: DEBUG nova.network.neutron [req-423ba74e-a121-4254-8f9f-890993843b88 req-ef4f9218-f60f-4f7f-b5c3-753949f9971a service nova] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Refreshing network info cache for port 0192162f-7b9c-4e13-9f0e-51910b514bed {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 664.216877] env[63593]: DEBUG nova.compute.manager [req-53f52735-11cc-45e1-b0b1-945faf28b592 req-d5be10a2-c458-4bf0-8df4-080db84cfc96 service nova] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Received event network-vif-deleted-cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 664.221458] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 664.307026] env[63593]: ERROR nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0192162f-7b9c-4e13-9f0e-51910b514bed, please check neutron logs for more information. [ 664.307026] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 664.307026] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 664.307026] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 664.307026] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 664.307026] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 664.307026] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 664.307026] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 664.307026] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 664.307026] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 664.307026] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 664.307026] env[63593]: ERROR nova.compute.manager raise self.value [ 664.307026] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 664.307026] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 664.307026] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 664.307026] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 664.307487] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 664.307487] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 664.307487] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0192162f-7b9c-4e13-9f0e-51910b514bed, please check neutron logs for more information. [ 664.307487] env[63593]: ERROR nova.compute.manager [ 664.307487] env[63593]: Traceback (most recent call last): [ 664.307487] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 664.307487] env[63593]: listener.cb(fileno) [ 664.307487] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 664.307487] env[63593]: result = function(*args, **kwargs) [ 664.307487] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 664.307487] env[63593]: return func(*args, **kwargs) [ 664.307487] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 664.307487] env[63593]: raise e [ 664.307487] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 664.307487] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 664.307487] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 664.307487] env[63593]: created_port_ids = self._update_ports_for_instance( [ 664.307487] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 664.307487] env[63593]: with excutils.save_and_reraise_exception(): [ 664.307487] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 664.307487] env[63593]: self.force_reraise() [ 664.307487] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 664.307487] env[63593]: raise self.value [ 664.307487] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 664.307487] env[63593]: updated_port = self._update_port( [ 664.307487] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 664.307487] env[63593]: _ensure_no_port_binding_failure(port) [ 664.307487] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 664.307487] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 664.308339] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 0192162f-7b9c-4e13-9f0e-51910b514bed, please check neutron logs for more information. [ 664.308339] env[63593]: Removing descriptor: 21 [ 664.308339] env[63593]: ERROR nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0192162f-7b9c-4e13-9f0e-51910b514bed, please check neutron logs for more information. [ 664.308339] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Traceback (most recent call last): [ 664.308339] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 664.308339] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] yield resources [ 664.308339] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 664.308339] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self.driver.spawn(context, instance, image_meta, [ 664.308339] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 664.308339] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 664.308339] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 664.308339] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] vm_ref = self.build_virtual_machine(instance, [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] vif_infos = vmwarevif.get_vif_info(self._session, [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] for vif in network_info: [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] return self._sync_wrapper(fn, *args, **kwargs) [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self.wait() [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self[:] = self._gt.wait() [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] return self._exit_event.wait() [ 664.308749] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] result = hub.switch() [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] return self.greenlet.switch() [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] result = function(*args, **kwargs) [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] return func(*args, **kwargs) [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] raise e [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] nwinfo = self.network_api.allocate_for_instance( [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 664.309166] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] created_port_ids = self._update_ports_for_instance( [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] with excutils.save_and_reraise_exception(): [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self.force_reraise() [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] raise self.value [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] updated_port = self._update_port( [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] _ensure_no_port_binding_failure(port) [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 664.309585] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] raise exception.PortBindingFailed(port_id=port['id']) [ 664.309977] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] nova.exception.PortBindingFailed: Binding failed for port 0192162f-7b9c-4e13-9f0e-51910b514bed, please check neutron logs for more information. [ 664.309977] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] [ 664.309977] env[63593]: INFO nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Terminating instance [ 664.312237] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Acquiring lock "refresh_cache-4869e3e4-48bb-446f-a15e-3a53bae1bc9b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 664.507855] env[63593]: DEBUG nova.compute.utils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 664.509982] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 664.509982] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 664.627860] env[63593]: DEBUG nova.policy [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f66fbf8fe6a645ed84fa3dd55175aa7c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3889b4d282fd4033a9071d2c37a863f7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 664.723758] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "refresh_cache-bb3a0a79-ef74-4fb0-a286-91b1b92df751" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 664.724406] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 664.724500] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 664.724778] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d8d9436c-a689-4adf-8242-2be84affc8c2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 664.734531] env[63593]: DEBUG nova.network.neutron [req-423ba74e-a121-4254-8f9f-890993843b88 req-ef4f9218-f60f-4f7f-b5c3-753949f9971a service nova] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 664.746797] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57725eb2-8d3c-4c7b-8ed2-7da65c999116 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 664.786340] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance bb3a0a79-ef74-4fb0-a286-91b1b92df751 could not be found. [ 664.786340] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 664.786340] env[63593]: INFO nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Took 0.06 seconds to destroy the instance on the hypervisor. [ 664.786340] env[63593]: DEBUG oslo.service.loopingcall [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 664.790029] env[63593]: DEBUG nova.compute.manager [-] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 664.790154] env[63593]: DEBUG nova.network.neutron [-] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 664.817457] env[63593]: DEBUG nova.network.neutron [-] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 664.940429] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13640590-82eb-4c31-ac43-ec1d5638582a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 664.948473] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd4e94f8-de98-41b7-a191-57601ee8d43b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 664.980286] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f997fe9-982a-44f4-8eca-ffd2faf25c0b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 664.988614] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66b428ca-e4df-475c-9c17-f5360a113335 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.003375] env[63593]: DEBUG nova.compute.provider_tree [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 665.013028] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 665.027145] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Successfully created port: ea563e31-7d0e-4720-b567-e7cef236d61f {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 665.044382] env[63593]: DEBUG nova.network.neutron [req-423ba74e-a121-4254-8f9f-890993843b88 req-ef4f9218-f60f-4f7f-b5c3-753949f9971a service nova] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 665.321354] env[63593]: DEBUG nova.network.neutron [-] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 665.507119] env[63593]: DEBUG nova.scheduler.client.report [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 665.547603] env[63593]: DEBUG oslo_concurrency.lockutils [req-423ba74e-a121-4254-8f9f-890993843b88 req-ef4f9218-f60f-4f7f-b5c3-753949f9971a service nova] Releasing lock "refresh_cache-4869e3e4-48bb-446f-a15e-3a53bae1bc9b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 665.548013] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Acquired lock "refresh_cache-4869e3e4-48bb-446f-a15e-3a53bae1bc9b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 665.548230] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 665.826556] env[63593]: INFO nova.compute.manager [-] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Took 1.04 seconds to deallocate network for instance. [ 665.830110] env[63593]: DEBUG nova.compute.claims [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 665.830391] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 666.015226] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.017s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 666.015900] env[63593]: ERROR nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 34edec86-cf21-4cd5-8dfd-219a396ba46b, please check neutron logs for more information. [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Traceback (most recent call last): [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self.driver.spawn(context, instance, image_meta, [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] vm_ref = self.build_virtual_machine(instance, [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] vif_infos = vmwarevif.get_vif_info(self._session, [ 666.015900] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] for vif in network_info: [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] return self._sync_wrapper(fn, *args, **kwargs) [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self.wait() [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self[:] = self._gt.wait() [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] return self._exit_event.wait() [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] result = hub.switch() [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 666.016435] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] return self.greenlet.switch() [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] result = function(*args, **kwargs) [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] return func(*args, **kwargs) [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] raise e [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] nwinfo = self.network_api.allocate_for_instance( [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] created_port_ids = self._update_ports_for_instance( [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] with excutils.save_and_reraise_exception(): [ 666.017449] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] self.force_reraise() [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] raise self.value [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] updated_port = self._update_port( [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] _ensure_no_port_binding_failure(port) [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] raise exception.PortBindingFailed(port_id=port['id']) [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] nova.exception.PortBindingFailed: Binding failed for port 34edec86-cf21-4cd5-8dfd-219a396ba46b, please check neutron logs for more information. [ 666.018189] env[63593]: ERROR nova.compute.manager [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] [ 666.019296] env[63593]: DEBUG nova.compute.utils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Binding failed for port 34edec86-cf21-4cd5-8dfd-219a396ba46b, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 666.019296] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 10.772s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 666.026406] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Build of instance 0dbcfda1-a081-4299-a117-3a1974e2510c was re-scheduled: Binding failed for port 34edec86-cf21-4cd5-8dfd-219a396ba46b, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 666.026885] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 666.028016] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Acquiring lock "refresh_cache-0dbcfda1-a081-4299-a117-3a1974e2510c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 666.028016] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Acquired lock "refresh_cache-0dbcfda1-a081-4299-a117-3a1974e2510c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 666.028016] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 666.029132] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 666.063900] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 666.064118] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 666.064273] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 666.064447] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 666.064586] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 666.064725] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 666.064925] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 666.065093] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 666.065260] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 666.065418] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 666.065585] env[63593]: DEBUG nova.virt.hardware [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 666.066601] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2cf6b63c-9781-4190-9df0-91b6b992dcea {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.077805] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32f1face-2a78-48b3-9293-f89f2893de7c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.083758] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 666.289427] env[63593]: DEBUG nova.compute.manager [req-f34d1792-6d9a-4499-a326-7b7d38391626 req-26425522-1123-45f9-9eb2-5caffdaaccfd service nova] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Received event network-vif-deleted-0192162f-7b9c-4e13-9f0e-51910b514bed {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 666.350699] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 666.565642] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 666.853999] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Releasing lock "refresh_cache-4869e3e4-48bb-446f-a15e-3a53bae1bc9b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 666.855452] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 666.857108] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 666.857108] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bceb8fd5-7dbd-4cb7-8a8c-3ae7d1fbeeb8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.869333] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-600b89ea-4bb6-4fc3-a757-b5758b74163b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.903029] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4869e3e4-48bb-446f-a15e-3a53bae1bc9b could not be found. [ 666.903265] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 666.903458] env[63593]: INFO nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Took 0.05 seconds to destroy the instance on the hypervisor. [ 666.903708] env[63593]: DEBUG oslo.service.loopingcall [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 666.907693] env[63593]: DEBUG nova.compute.manager [-] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 666.907806] env[63593]: DEBUG nova.network.neutron [-] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 666.938198] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55e3d6df-09eb-41ce-9f0a-a490c72b6d94 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.945468] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de9b479d-b37a-4b6c-90c3-f5bce1209071 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.950061] env[63593]: DEBUG nova.network.neutron [-] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 666.983982] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 666.985612] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ddba225d-fa42-415d-bb1e-ef6efffd82cb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 666.993069] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8648023f-32de-4951-9ebb-b03702a74c62 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 667.008065] env[63593]: DEBUG nova.compute.provider_tree [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 667.217030] env[63593]: ERROR nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ea563e31-7d0e-4720-b567-e7cef236d61f, please check neutron logs for more information. [ 667.217030] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 667.217030] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 667.217030] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 667.217030] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 667.217030] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 667.217030] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 667.217030] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 667.217030] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 667.217030] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 667.217030] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 667.217030] env[63593]: ERROR nova.compute.manager raise self.value [ 667.217030] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 667.217030] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 667.217030] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 667.217030] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 667.217550] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 667.217550] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 667.217550] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ea563e31-7d0e-4720-b567-e7cef236d61f, please check neutron logs for more information. [ 667.217550] env[63593]: ERROR nova.compute.manager [ 667.217550] env[63593]: Traceback (most recent call last): [ 667.217550] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 667.217550] env[63593]: listener.cb(fileno) [ 667.217550] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 667.217550] env[63593]: result = function(*args, **kwargs) [ 667.217550] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 667.217550] env[63593]: return func(*args, **kwargs) [ 667.217550] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 667.217550] env[63593]: raise e [ 667.217550] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 667.217550] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 667.217550] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 667.217550] env[63593]: created_port_ids = self._update_ports_for_instance( [ 667.217550] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 667.217550] env[63593]: with excutils.save_and_reraise_exception(): [ 667.217550] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 667.217550] env[63593]: self.force_reraise() [ 667.217550] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 667.217550] env[63593]: raise self.value [ 667.217550] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 667.217550] env[63593]: updated_port = self._update_port( [ 667.217550] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 667.217550] env[63593]: _ensure_no_port_binding_failure(port) [ 667.217550] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 667.217550] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 667.218385] env[63593]: nova.exception.PortBindingFailed: Binding failed for port ea563e31-7d0e-4720-b567-e7cef236d61f, please check neutron logs for more information. [ 667.218385] env[63593]: Removing descriptor: 21 [ 667.218385] env[63593]: ERROR nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ea563e31-7d0e-4720-b567-e7cef236d61f, please check neutron logs for more information. [ 667.218385] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Traceback (most recent call last): [ 667.218385] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 667.218385] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] yield resources [ 667.218385] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 667.218385] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self.driver.spawn(context, instance, image_meta, [ 667.218385] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 667.218385] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self._vmops.spawn(context, instance, image_meta, injected_files, [ 667.218385] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 667.218385] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] vm_ref = self.build_virtual_machine(instance, [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] vif_infos = vmwarevif.get_vif_info(self._session, [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] for vif in network_info: [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] return self._sync_wrapper(fn, *args, **kwargs) [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self.wait() [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self[:] = self._gt.wait() [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] return self._exit_event.wait() [ 667.218729] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] result = hub.switch() [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] return self.greenlet.switch() [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] result = function(*args, **kwargs) [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] return func(*args, **kwargs) [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] raise e [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] nwinfo = self.network_api.allocate_for_instance( [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 667.219098] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] created_port_ids = self._update_ports_for_instance( [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] with excutils.save_and_reraise_exception(): [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self.force_reraise() [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] raise self.value [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] updated_port = self._update_port( [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] _ensure_no_port_binding_failure(port) [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 667.219523] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] raise exception.PortBindingFailed(port_id=port['id']) [ 667.219874] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] nova.exception.PortBindingFailed: Binding failed for port ea563e31-7d0e-4720-b567-e7cef236d61f, please check neutron logs for more information. [ 667.219874] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] [ 667.219874] env[63593]: INFO nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Terminating instance [ 667.221375] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "refresh_cache-d46033b4-b3d5-494c-b25c-106475fe4409" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 667.221375] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquired lock "refresh_cache-d46033b4-b3d5-494c-b25c-106475fe4409" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 667.221375] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 667.456584] env[63593]: DEBUG nova.network.neutron [-] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 667.489479] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Releasing lock "refresh_cache-0dbcfda1-a081-4299-a117-3a1974e2510c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 667.489951] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 667.490219] env[63593]: DEBUG nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 667.490431] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 667.512362] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 667.516033] env[63593]: DEBUG nova.scheduler.client.report [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 667.749155] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 667.865712] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 667.960262] env[63593]: INFO nova.compute.manager [-] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Took 1.05 seconds to deallocate network for instance. [ 667.964831] env[63593]: DEBUG nova.compute.claims [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 667.964831] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 668.015822] env[63593]: DEBUG nova.network.neutron [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 668.022896] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.005s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 668.023585] env[63593]: ERROR nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c5eddc15-1383-4b48-be39-d5589ce77368, please check neutron logs for more information. [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Traceback (most recent call last): [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self.driver.spawn(context, instance, image_meta, [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self._vmops.spawn(context, instance, image_meta, injected_files, [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] vm_ref = self.build_virtual_machine(instance, [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] vif_infos = vmwarevif.get_vif_info(self._session, [ 668.023585] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] for vif in network_info: [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] return self._sync_wrapper(fn, *args, **kwargs) [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self.wait() [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self[:] = self._gt.wait() [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] return self._exit_event.wait() [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] result = hub.switch() [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 668.023946] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] return self.greenlet.switch() [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] result = function(*args, **kwargs) [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] return func(*args, **kwargs) [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] raise e [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] nwinfo = self.network_api.allocate_for_instance( [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] created_port_ids = self._update_ports_for_instance( [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] with excutils.save_and_reraise_exception(): [ 668.024442] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] self.force_reraise() [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] raise self.value [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] updated_port = self._update_port( [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] _ensure_no_port_binding_failure(port) [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] raise exception.PortBindingFailed(port_id=port['id']) [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] nova.exception.PortBindingFailed: Binding failed for port c5eddc15-1383-4b48-be39-d5589ce77368, please check neutron logs for more information. [ 668.024837] env[63593]: ERROR nova.compute.manager [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] [ 668.025161] env[63593]: DEBUG nova.compute.utils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Binding failed for port c5eddc15-1383-4b48-be39-d5589ce77368, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 668.028779] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.261s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 668.034342] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Build of instance 81b0b694-b9fb-4416-8751-7f1e24e96869 was re-scheduled: Binding failed for port c5eddc15-1383-4b48-be39-d5589ce77368, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 668.034342] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 668.034342] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "refresh_cache-81b0b694-b9fb-4416-8751-7f1e24e96869" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 668.034342] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquired lock "refresh_cache-81b0b694-b9fb-4416-8751-7f1e24e96869" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 668.034630] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 668.373096] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Releasing lock "refresh_cache-d46033b4-b3d5-494c-b25c-106475fe4409" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 668.373652] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 668.373919] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 668.374222] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cc0952cb-6bf5-44b5-8250-3bb310ed6f58 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.393664] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9c85ba7-1c01-438b-82d2-3dd142c64854 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.424472] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d46033b4-b3d5-494c-b25c-106475fe4409 could not be found. [ 668.426014] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 668.426206] env[63593]: INFO nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Took 0.05 seconds to destroy the instance on the hypervisor. [ 668.426996] env[63593]: DEBUG oslo.service.loopingcall [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 668.431035] env[63593]: DEBUG nova.compute.manager [-] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 668.431035] env[63593]: DEBUG nova.network.neutron [-] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 668.469878] env[63593]: DEBUG nova.network.neutron [-] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 668.519006] env[63593]: INFO nova.compute.manager [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] [instance: 0dbcfda1-a081-4299-a117-3a1974e2510c] Took 1.03 seconds to deallocate network for instance. [ 668.581271] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 668.759997] env[63593]: DEBUG nova.compute.manager [req-9010f59d-3603-4034-afa3-3cd1f3daa95e req-04875cbe-49ec-4a63-94ca-d2a7c0093040 service nova] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Received event network-changed-ea563e31-7d0e-4720-b567-e7cef236d61f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 668.760210] env[63593]: DEBUG nova.compute.manager [req-9010f59d-3603-4034-afa3-3cd1f3daa95e req-04875cbe-49ec-4a63-94ca-d2a7c0093040 service nova] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Refreshing instance network info cache due to event network-changed-ea563e31-7d0e-4720-b567-e7cef236d61f. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 668.760425] env[63593]: DEBUG oslo_concurrency.lockutils [req-9010f59d-3603-4034-afa3-3cd1f3daa95e req-04875cbe-49ec-4a63-94ca-d2a7c0093040 service nova] Acquiring lock "refresh_cache-d46033b4-b3d5-494c-b25c-106475fe4409" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 668.760585] env[63593]: DEBUG oslo_concurrency.lockutils [req-9010f59d-3603-4034-afa3-3cd1f3daa95e req-04875cbe-49ec-4a63-94ca-d2a7c0093040 service nova] Acquired lock "refresh_cache-d46033b4-b3d5-494c-b25c-106475fe4409" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 668.760713] env[63593]: DEBUG nova.network.neutron [req-9010f59d-3603-4034-afa3-3cd1f3daa95e req-04875cbe-49ec-4a63-94ca-d2a7c0093040 service nova] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Refreshing network info cache for port ea563e31-7d0e-4720-b567-e7cef236d61f {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 668.841086] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 668.963451] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a0e1dfd-a4d5-419a-aee9-c3bacee0859a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.971426] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-972d14a6-a252-43a9-b79c-68e06eec47b2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.975418] env[63593]: DEBUG nova.network.neutron [-] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 669.007256] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4caf6e77-706b-4e6d-8c61-b0e4929987e1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 669.013024] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Acquiring lock "4f1c0deb-5759-44d2-a2ef-f94e1905949d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 669.013267] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Lock "4f1c0deb-5759-44d2-a2ef-f94e1905949d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 669.019131] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e099f4e-d99d-444c-9684-57fbf1c51bd8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 669.037480] env[63593]: DEBUG nova.compute.provider_tree [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 669.341569] env[63593]: DEBUG nova.network.neutron [req-9010f59d-3603-4034-afa3-3cd1f3daa95e req-04875cbe-49ec-4a63-94ca-d2a7c0093040 service nova] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 669.347654] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Releasing lock "refresh_cache-81b0b694-b9fb-4416-8751-7f1e24e96869" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 669.348015] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 669.348307] env[63593]: DEBUG nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 669.348535] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 669.402021] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 669.478453] env[63593]: INFO nova.compute.manager [-] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Took 1.05 seconds to deallocate network for instance. [ 669.484795] env[63593]: DEBUG nova.compute.claims [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 669.484795] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 669.518841] env[63593]: DEBUG nova.network.neutron [req-9010f59d-3603-4034-afa3-3cd1f3daa95e req-04875cbe-49ec-4a63-94ca-d2a7c0093040 service nova] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 669.539915] env[63593]: DEBUG nova.scheduler.client.report [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 669.567817] env[63593]: INFO nova.scheduler.client.report [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Deleted allocations for instance 0dbcfda1-a081-4299-a117-3a1974e2510c [ 669.910072] env[63593]: DEBUG nova.network.neutron [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 670.021250] env[63593]: DEBUG oslo_concurrency.lockutils [req-9010f59d-3603-4034-afa3-3cd1f3daa95e req-04875cbe-49ec-4a63-94ca-d2a7c0093040 service nova] Releasing lock "refresh_cache-d46033b4-b3d5-494c-b25c-106475fe4409" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 670.021250] env[63593]: DEBUG nova.compute.manager [req-9010f59d-3603-4034-afa3-3cd1f3daa95e req-04875cbe-49ec-4a63-94ca-d2a7c0093040 service nova] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Received event network-vif-deleted-ea563e31-7d0e-4720-b567-e7cef236d61f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 670.048148] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.022s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 670.048726] env[63593]: ERROR nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 670.048726] env[63593]: Faults: ['InvalidArgument'] [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Traceback (most recent call last): [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] self.driver.spawn(context, instance, image_meta, [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] self._vmops.spawn(context, instance, image_meta, injected_files, [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] self._fetch_image_if_missing(context, vi) [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] image_cache(vi, tmp_image_ds_loc) [ 670.048726] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] vm_util.copy_virtual_disk( [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] session._wait_for_task(vmdk_copy_task) [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] return self.wait_for_task(task_ref) [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] return evt.wait() [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] result = hub.switch() [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] return self.greenlet.switch() [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 670.049177] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] self.f(*self.args, **self.kw) [ 670.049563] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 670.049563] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] raise exceptions.translate_fault(task_info.error) [ 670.049563] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 670.049563] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Faults: ['InvalidArgument'] [ 670.049563] env[63593]: ERROR nova.compute.manager [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] [ 670.051440] env[63593]: DEBUG nova.compute.utils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 670.053444] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.281s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 670.060974] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Build of instance 7044b799-74ed-4711-9b53-08f8a8ddbb78 was re-scheduled: A specified parameter was not correct: fileType [ 670.060974] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 670.060974] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 670.060974] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquiring lock "refresh_cache-7044b799-74ed-4711-9b53-08f8a8ddbb78" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 670.060974] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Acquired lock "refresh_cache-7044b799-74ed-4711-9b53-08f8a8ddbb78" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 670.060974] env[63593]: DEBUG nova.network.neutron [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 670.078271] env[63593]: DEBUG oslo_concurrency.lockutils [None req-784d5026-a058-45f9-9b50-a2a58513dfea tempest-ServerMetadataTestJSON-1443551695 tempest-ServerMetadataTestJSON-1443551695-project-member] Lock "0dbcfda1-a081-4299-a117-3a1974e2510c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 58.501s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 670.420033] env[63593]: INFO nova.compute.manager [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 81b0b694-b9fb-4416-8751-7f1e24e96869] Took 1.07 seconds to deallocate network for instance. [ 670.581957] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 670.600407] env[63593]: DEBUG nova.network.neutron [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 670.674269] env[63593]: DEBUG nova.network.neutron [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 670.832835] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "4da891e9-39e0-4739-b1b9-fb438c1111ab" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.832835] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "4da891e9-39e0-4739-b1b9-fb438c1111ab" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 670.899890] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "788e5910-c189-4d13-a2fa-e5d6e915e95d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.900280] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "788e5910-c189-4d13-a2fa-e5d6e915e95d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 671.104819] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bf78f07-e61a-4ed6-8480-870be50c9d54 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 671.109442] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 671.115703] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b409a7fc-a7d4-4291-9f63-435d21120d1c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 671.150371] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dee54b8f-9069-48b5-8425-c0dcde6428f6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 671.162497] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d6ff608-5669-4a34-a0f6-c0b05e7e9404 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 671.178454] env[63593]: DEBUG nova.compute.provider_tree [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 671.180096] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Releasing lock "refresh_cache-7044b799-74ed-4711-9b53-08f8a8ddbb78" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 671.180301] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 671.180534] env[63593]: DEBUG nova.compute.manager [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 671.475700] env[63593]: INFO nova.scheduler.client.report [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Deleted allocations for instance 81b0b694-b9fb-4416-8751-7f1e24e96869 [ 671.686052] env[63593]: DEBUG nova.scheduler.client.report [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 671.853971] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Acquiring lock "b9bd0d9b-322f-476a-ae3f-792ccd869529" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 671.854308] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Lock "b9bd0d9b-322f-476a-ae3f-792ccd869529" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 671.996102] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fcaab421-24f5-4f56-bf23-f48ee6225abb tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "81b0b694-b9fb-4416-8751-7f1e24e96869" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 59.103s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 672.191460] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.139s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 672.195020] env[63593]: ERROR nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 289407e8-c816-4ada-a146-53327b567f6c, please check neutron logs for more information. [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Traceback (most recent call last): [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self.driver.spawn(context, instance, image_meta, [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] vm_ref = self.build_virtual_machine(instance, [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] vif_infos = vmwarevif.get_vif_info(self._session, [ 672.195020] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] for vif in network_info: [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] return self._sync_wrapper(fn, *args, **kwargs) [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self.wait() [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self[:] = self._gt.wait() [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] return self._exit_event.wait() [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] result = hub.switch() [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 672.195545] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] return self.greenlet.switch() [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] result = function(*args, **kwargs) [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] return func(*args, **kwargs) [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] raise e [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] nwinfo = self.network_api.allocate_for_instance( [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] created_port_ids = self._update_ports_for_instance( [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] with excutils.save_and_reraise_exception(): [ 672.196063] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] self.force_reraise() [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] raise self.value [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] updated_port = self._update_port( [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] _ensure_no_port_binding_failure(port) [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] raise exception.PortBindingFailed(port_id=port['id']) [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] nova.exception.PortBindingFailed: Binding failed for port 289407e8-c816-4ada-a146-53327b567f6c, please check neutron logs for more information. [ 672.196510] env[63593]: ERROR nova.compute.manager [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] [ 672.196834] env[63593]: DEBUG nova.compute.utils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Binding failed for port 289407e8-c816-4ada-a146-53327b567f6c, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 672.196834] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.802s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 672.205119] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Build of instance c6d88cbe-488b-490b-a37c-51a311276dfe was re-scheduled: Binding failed for port 289407e8-c816-4ada-a146-53327b567f6c, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 672.205898] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 672.206411] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Acquiring lock "refresh_cache-c6d88cbe-488b-490b-a37c-51a311276dfe" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 672.206684] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Acquired lock "refresh_cache-c6d88cbe-488b-490b-a37c-51a311276dfe" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 672.206957] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 672.239018] env[63593]: INFO nova.scheduler.client.report [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Deleted allocations for instance 7044b799-74ed-4711-9b53-08f8a8ddbb78 [ 672.499222] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 672.747963] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8be8c36e-61d8-4699-b301-fbc563fc929c tempest-ServersAdmin275Test-207689732 tempest-ServersAdmin275Test-207689732-project-member] Lock "7044b799-74ed-4711-9b53-08f8a8ddbb78" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 76.775s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 672.749539] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "7044b799-74ed-4711-9b53-08f8a8ddbb78" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 71.717s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 672.749734] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 7044b799-74ed-4711-9b53-08f8a8ddbb78] During sync_power_state the instance has a pending task (block_device_mapping). Skip. [ 672.749901] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "7044b799-74ed-4711-9b53-08f8a8ddbb78" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 672.771075] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 672.989190] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 673.045207] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 673.214118] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a2b2527-8cd8-41f4-a974-847ea1d95f44 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 673.224547] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d641121-74e3-480f-875f-4b4721aa69c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 673.266559] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 673.271933] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2ad58c6-4423-4a7c-a7ff-1df33a2ea5e9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 673.280448] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9cfde7f6-3483-4ed8-933a-d2f622a2e187 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 673.292638] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "9bc65ce3-6459-4356-8a63-74f55b6f3b47" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 673.292638] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "9bc65ce3-6459-4356-8a63-74f55b6f3b47" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 673.302048] env[63593]: DEBUG nova.compute.provider_tree [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 673.494316] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Releasing lock "refresh_cache-c6d88cbe-488b-490b-a37c-51a311276dfe" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 673.494316] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 673.494316] env[63593]: DEBUG nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 673.494316] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 673.515708] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 673.791257] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 673.805117] env[63593]: DEBUG nova.scheduler.client.report [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 674.018252] env[63593]: DEBUG nova.network.neutron [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 674.135158] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "b62cc771-973a-4d23-87b4-825211494d9c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.135363] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "b62cc771-973a-4d23-87b4-825211494d9c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.314017] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.118s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 674.314017] env[63593]: ERROR nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port bcb30ba2-9252-4597-bc57-0c9e2853ec60, please check neutron logs for more information. [ 674.314017] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Traceback (most recent call last): [ 674.314017] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 674.314017] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self.driver.spawn(context, instance, image_meta, [ 674.314017] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 674.314017] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 674.314017] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 674.314017] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] vm_ref = self.build_virtual_machine(instance, [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] vif_infos = vmwarevif.get_vif_info(self._session, [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] for vif in network_info: [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] return self._sync_wrapper(fn, *args, **kwargs) [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self.wait() [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self[:] = self._gt.wait() [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] return self._exit_event.wait() [ 674.314288] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] result = hub.switch() [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] return self.greenlet.switch() [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] result = function(*args, **kwargs) [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] return func(*args, **kwargs) [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] raise e [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] nwinfo = self.network_api.allocate_for_instance( [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 674.314660] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] created_port_ids = self._update_ports_for_instance( [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] with excutils.save_and_reraise_exception(): [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] self.force_reraise() [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] raise self.value [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] updated_port = self._update_port( [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] _ensure_no_port_binding_failure(port) [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 674.314970] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] raise exception.PortBindingFailed(port_id=port['id']) [ 674.315268] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] nova.exception.PortBindingFailed: Binding failed for port bcb30ba2-9252-4597-bc57-0c9e2853ec60, please check neutron logs for more information. [ 674.315268] env[63593]: ERROR nova.compute.manager [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] [ 674.315268] env[63593]: DEBUG nova.compute.utils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Binding failed for port bcb30ba2-9252-4597-bc57-0c9e2853ec60, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 674.315472] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.128s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.317559] env[63593]: INFO nova.compute.claims [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 674.320392] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Build of instance 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1 was re-scheduled: Binding failed for port bcb30ba2-9252-4597-bc57-0c9e2853ec60, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 674.322030] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 674.322030] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Acquiring lock "refresh_cache-6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 674.322030] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Acquired lock "refresh_cache-6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 674.322030] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 674.522456] env[63593]: INFO nova.compute.manager [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] [instance: c6d88cbe-488b-490b-a37c-51a311276dfe] Took 1.03 seconds to deallocate network for instance. [ 674.845039] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 674.949835] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 675.055982] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "b965361a-6b76-4836-9887-b59ca990fd67" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 675.056326] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "b965361a-6b76-4836-9887-b59ca990fd67" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 675.134663] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "077acb40-d01b-42ad-aa68-cbffc4087100" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 675.137303] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "077acb40-d01b-42ad-aa68-cbffc4087100" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.003s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 675.451803] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Releasing lock "refresh_cache-6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 675.452268] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 675.452639] env[63593]: DEBUG nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 675.452952] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 675.475134] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 675.558420] env[63593]: INFO nova.scheduler.client.report [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Deleted allocations for instance c6d88cbe-488b-490b-a37c-51a311276dfe [ 675.837009] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e98fd6e-9093-49cd-870b-d7d7bacbb176 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 675.845141] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f4a9f66-8880-4a73-a369-860c78ca067d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 675.878846] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf7c66af-e7b9-416c-9158-87fc71b02c1f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 675.888309] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eac997b1-eb86-49b2-9a8e-8ebb0095f6cd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 675.902284] env[63593]: DEBUG nova.compute.provider_tree [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 675.978662] env[63593]: DEBUG nova.network.neutron [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 676.067373] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d09243a9-3287-493d-b973-7a3bb6de6c9f tempest-ServersWithSpecificFlavorTestJSON-724607427 tempest-ServersWithSpecificFlavorTestJSON-724607427-project-member] Lock "c6d88cbe-488b-490b-a37c-51a311276dfe" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 62.488s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 676.407138] env[63593]: DEBUG nova.scheduler.client.report [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 676.481585] env[63593]: INFO nova.compute.manager [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] [instance: 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1] Took 1.03 seconds to deallocate network for instance. [ 676.570630] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 676.912400] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.597s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 676.912913] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 676.915550] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.085s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 677.098141] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 677.420178] env[63593]: DEBUG nova.compute.utils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 677.425938] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 677.425938] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 677.485811] env[63593]: DEBUG nova.policy [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '1db27961398346bb9912842b78a2ce98', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7acf74f1027c4081a3986a6e3e7d270e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 677.513187] env[63593]: INFO nova.scheduler.client.report [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Deleted allocations for instance 6d586ded-7e1a-426f-9a26-eebcc3b3b9a1 [ 677.771717] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6347df71-b4b8-4220-abcd-f74afde27a07 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 677.779674] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a747124e-bac2-4a1e-9401-7757f1682de0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 677.810908] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ce8fa6c-8a2b-4ea2-aaf1-53f99f7b9c5c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 677.818478] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a81c84b-f552-4483-8804-7f0985876935 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 677.831233] env[63593]: DEBUG nova.compute.provider_tree [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 677.926353] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 677.976210] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Successfully created port: 772db4f5-aac3-466d-a93a-3fa23280b635 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 678.023839] env[63593]: DEBUG oslo_concurrency.lockutils [None req-26013df1-62ca-4062-84e1-3a9c60a31e19 tempest-ServerDiagnosticsTest-1427930250 tempest-ServerDiagnosticsTest-1427930250-project-member] Lock "6d586ded-7e1a-426f-9a26-eebcc3b3b9a1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 63.154s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 678.333854] env[63593]: DEBUG nova.scheduler.client.report [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 678.525769] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 678.570779] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "1db3e15b-3168-4d01-930f-fe8008b95df6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 678.571009] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "1db3e15b-3168-4d01-930f-fe8008b95df6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 678.843020] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.924s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 678.843020] env[63593]: ERROR nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c, please check neutron logs for more information. [ 678.843020] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Traceback (most recent call last): [ 678.843020] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 678.843020] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self.driver.spawn(context, instance, image_meta, [ 678.843020] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 678.843020] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self._vmops.spawn(context, instance, image_meta, injected_files, [ 678.843020] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 678.843020] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] vm_ref = self.build_virtual_machine(instance, [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] vif_infos = vmwarevif.get_vif_info(self._session, [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] for vif in network_info: [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] return self._sync_wrapper(fn, *args, **kwargs) [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self.wait() [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self[:] = self._gt.wait() [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] return self._exit_event.wait() [ 678.843437] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] result = hub.switch() [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] return self.greenlet.switch() [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] result = function(*args, **kwargs) [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] return func(*args, **kwargs) [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] raise e [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] nwinfo = self.network_api.allocate_for_instance( [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 678.843767] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] created_port_ids = self._update_ports_for_instance( [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] with excutils.save_and_reraise_exception(): [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] self.force_reraise() [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] raise self.value [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] updated_port = self._update_port( [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] _ensure_no_port_binding_failure(port) [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 678.844095] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] raise exception.PortBindingFailed(port_id=port['id']) [ 678.844399] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] nova.exception.PortBindingFailed: Binding failed for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c, please check neutron logs for more information. [ 678.844399] env[63593]: ERROR nova.compute.manager [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] [ 678.844399] env[63593]: DEBUG nova.compute.utils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Binding failed for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 678.844399] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 10.878s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 678.847814] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Build of instance bb3a0a79-ef74-4fb0-a286-91b1b92df751 was re-scheduled: Binding failed for port cc172e31-e3b6-49e2-9b1c-aa9f8be0ed9c, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 678.847814] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 678.848103] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "refresh_cache-bb3a0a79-ef74-4fb0-a286-91b1b92df751" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 678.848357] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "refresh_cache-bb3a0a79-ef74-4fb0-a286-91b1b92df751" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 678.848595] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 678.939917] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 678.973235] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 678.973575] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 678.974664] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 678.974664] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 678.974664] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 678.974664] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 678.974835] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 678.975075] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 678.975287] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 678.975639] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 678.975842] env[63593]: DEBUG nova.virt.hardware [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 678.976851] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75d8e078-9d12-4df9-bfd4-095b26f50d41 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 678.986828] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c2a4eea-ae33-47b4-9181-c870ba47e22d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 679.052839] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 679.393027] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 679.683828] env[63593]: DEBUG nova.compute.manager [req-32f2777e-b6a0-41e3-8b33-3ff4d32e9e57 req-c02bd391-67b0-497e-91e5-012bc6cd35ae service nova] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Received event network-changed-772db4f5-aac3-466d-a93a-3fa23280b635 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 679.683942] env[63593]: DEBUG nova.compute.manager [req-32f2777e-b6a0-41e3-8b33-3ff4d32e9e57 req-c02bd391-67b0-497e-91e5-012bc6cd35ae service nova] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Refreshing instance network info cache due to event network-changed-772db4f5-aac3-466d-a93a-3fa23280b635. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 679.684192] env[63593]: DEBUG oslo_concurrency.lockutils [req-32f2777e-b6a0-41e3-8b33-3ff4d32e9e57 req-c02bd391-67b0-497e-91e5-012bc6cd35ae service nova] Acquiring lock "refresh_cache-cc41b8c4-c49a-4f23-bd16-ca2362a687d0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 679.685597] env[63593]: DEBUG oslo_concurrency.lockutils [req-32f2777e-b6a0-41e3-8b33-3ff4d32e9e57 req-c02bd391-67b0-497e-91e5-012bc6cd35ae service nova] Acquired lock "refresh_cache-cc41b8c4-c49a-4f23-bd16-ca2362a687d0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 679.685597] env[63593]: DEBUG nova.network.neutron [req-32f2777e-b6a0-41e3-8b33-3ff4d32e9e57 req-c02bd391-67b0-497e-91e5-012bc6cd35ae service nova] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Refreshing network info cache for port 772db4f5-aac3-466d-a93a-3fa23280b635 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 679.688241] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 679.752361] env[63593]: ERROR nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 772db4f5-aac3-466d-a93a-3fa23280b635, please check neutron logs for more information. [ 679.752361] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 679.752361] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 679.752361] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 679.752361] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 679.752361] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 679.752361] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 679.752361] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 679.752361] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 679.752361] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 679.752361] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 679.752361] env[63593]: ERROR nova.compute.manager raise self.value [ 679.752361] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 679.752361] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 679.752361] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 679.752361] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 679.752724] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 679.752724] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 679.752724] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 772db4f5-aac3-466d-a93a-3fa23280b635, please check neutron logs for more information. [ 679.752724] env[63593]: ERROR nova.compute.manager [ 679.752724] env[63593]: Traceback (most recent call last): [ 679.752724] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 679.752724] env[63593]: listener.cb(fileno) [ 679.752724] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 679.752724] env[63593]: result = function(*args, **kwargs) [ 679.752724] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 679.752724] env[63593]: return func(*args, **kwargs) [ 679.752724] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 679.752724] env[63593]: raise e [ 679.752724] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 679.752724] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 679.752724] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 679.752724] env[63593]: created_port_ids = self._update_ports_for_instance( [ 679.752724] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 679.752724] env[63593]: with excutils.save_and_reraise_exception(): [ 679.752724] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 679.752724] env[63593]: self.force_reraise() [ 679.752724] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 679.752724] env[63593]: raise self.value [ 679.752724] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 679.752724] env[63593]: updated_port = self._update_port( [ 679.752724] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 679.752724] env[63593]: _ensure_no_port_binding_failure(port) [ 679.752724] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 679.752724] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 679.753345] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 772db4f5-aac3-466d-a93a-3fa23280b635, please check neutron logs for more information. [ 679.753345] env[63593]: Removing descriptor: 21 [ 679.753663] env[63593]: ERROR nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 772db4f5-aac3-466d-a93a-3fa23280b635, please check neutron logs for more information. [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Traceback (most recent call last): [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] yield resources [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self.driver.spawn(context, instance, image_meta, [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] vm_ref = self.build_virtual_machine(instance, [ 679.753663] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] vif_infos = vmwarevif.get_vif_info(self._session, [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] for vif in network_info: [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] return self._sync_wrapper(fn, *args, **kwargs) [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self.wait() [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self[:] = self._gt.wait() [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] return self._exit_event.wait() [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 679.754030] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] result = hub.switch() [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] return self.greenlet.switch() [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] result = function(*args, **kwargs) [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] return func(*args, **kwargs) [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] raise e [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] nwinfo = self.network_api.allocate_for_instance( [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] created_port_ids = self._update_ports_for_instance( [ 679.754316] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] with excutils.save_and_reraise_exception(): [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self.force_reraise() [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] raise self.value [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] updated_port = self._update_port( [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] _ensure_no_port_binding_failure(port) [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] raise exception.PortBindingFailed(port_id=port['id']) [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] nova.exception.PortBindingFailed: Binding failed for port 772db4f5-aac3-466d-a93a-3fa23280b635, please check neutron logs for more information. [ 679.754596] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] [ 679.754878] env[63593]: INFO nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Terminating instance [ 679.756469] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "refresh_cache-cc41b8c4-c49a-4f23-bd16-ca2362a687d0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 679.782159] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a820833b-414b-4183-ae7b-3fc46315940d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 679.790034] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57b13a20-4295-4a63-bf36-3e7a40fbf0eb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 679.820593] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6e07c7b-ffcb-4e8e-9f8d-4c734609d6c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 679.829614] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df5f1df6-48ae-406f-bbce-5eb1fbd4b9c8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 679.845056] env[63593]: DEBUG nova.compute.provider_tree [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 680.193636] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "refresh_cache-bb3a0a79-ef74-4fb0-a286-91b1b92df751" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 680.193884] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 680.194030] env[63593]: DEBUG nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 680.194276] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 680.215485] env[63593]: DEBUG nova.network.neutron [req-32f2777e-b6a0-41e3-8b33-3ff4d32e9e57 req-c02bd391-67b0-497e-91e5-012bc6cd35ae service nova] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 680.324574] env[63593]: DEBUG nova.network.neutron [req-32f2777e-b6a0-41e3-8b33-3ff4d32e9e57 req-c02bd391-67b0-497e-91e5-012bc6cd35ae service nova] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 680.347750] env[63593]: DEBUG nova.scheduler.client.report [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 680.360018] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 680.830790] env[63593]: DEBUG oslo_concurrency.lockutils [req-32f2777e-b6a0-41e3-8b33-3ff4d32e9e57 req-c02bd391-67b0-497e-91e5-012bc6cd35ae service nova] Releasing lock "refresh_cache-cc41b8c4-c49a-4f23-bd16-ca2362a687d0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 680.832719] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquired lock "refresh_cache-cc41b8c4-c49a-4f23-bd16-ca2362a687d0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 680.832719] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 680.860719] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.018s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 680.862407] env[63593]: ERROR nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0192162f-7b9c-4e13-9f0e-51910b514bed, please check neutron logs for more information. [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Traceback (most recent call last): [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self.driver.spawn(context, instance, image_meta, [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] vm_ref = self.build_virtual_machine(instance, [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] vif_infos = vmwarevif.get_vif_info(self._session, [ 680.862407] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] for vif in network_info: [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] return self._sync_wrapper(fn, *args, **kwargs) [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self.wait() [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self[:] = self._gt.wait() [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] return self._exit_event.wait() [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] result = hub.switch() [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 680.863598] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] return self.greenlet.switch() [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] result = function(*args, **kwargs) [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] return func(*args, **kwargs) [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] raise e [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] nwinfo = self.network_api.allocate_for_instance( [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] created_port_ids = self._update_ports_for_instance( [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] with excutils.save_and_reraise_exception(): [ 680.863913] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] self.force_reraise() [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] raise self.value [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] updated_port = self._update_port( [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] _ensure_no_port_binding_failure(port) [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] raise exception.PortBindingFailed(port_id=port['id']) [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] nova.exception.PortBindingFailed: Binding failed for port 0192162f-7b9c-4e13-9f0e-51910b514bed, please check neutron logs for more information. [ 680.864288] env[63593]: ERROR nova.compute.manager [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] [ 680.864516] env[63593]: DEBUG nova.compute.utils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Binding failed for port 0192162f-7b9c-4e13-9f0e-51910b514bed, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 680.864516] env[63593]: DEBUG nova.network.neutron [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 680.866876] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Build of instance 4869e3e4-48bb-446f-a15e-3a53bae1bc9b was re-scheduled: Binding failed for port 0192162f-7b9c-4e13-9f0e-51910b514bed, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 680.867423] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 680.867564] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Acquiring lock "refresh_cache-4869e3e4-48bb-446f-a15e-3a53bae1bc9b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 680.867712] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Acquired lock "refresh_cache-4869e3e4-48bb-446f-a15e-3a53bae1bc9b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 680.867864] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 680.868735] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.384s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 681.371080] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 681.375369] env[63593]: INFO nova.compute.manager [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: bb3a0a79-ef74-4fb0-a286-91b1b92df751] Took 1.18 seconds to deallocate network for instance. [ 681.436558] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 681.528026] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 681.683682] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 681.787188] env[63593]: DEBUG nova.compute.manager [req-c494f100-ead3-4354-971e-45ebf58b2125 req-d0eab147-8f53-44c9-a922-be57dc62bdf2 service nova] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Received event network-vif-deleted-772db4f5-aac3-466d-a93a-3fa23280b635 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 681.914430] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f4b6304-b67c-4e25-8c0f-195ac63c7dad {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.923728] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43e9c86e-d20c-4297-b8de-092f944dca51 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.958273] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccf15ef3-c679-4eab-8a92-16400e9e4e28 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.965849] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Acquiring lock "b27b2ed8-ec99-4962-8742-3a8264906b47" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 681.966195] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Lock "b27b2ed8-ec99-4962-8742-3a8264906b47" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 681.970326] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-000e2859-0302-46b7-9443-a6de03fba9db {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.985133] env[63593]: DEBUG nova.compute.provider_tree [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 682.029310] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Releasing lock "refresh_cache-cc41b8c4-c49a-4f23-bd16-ca2362a687d0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 682.029849] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 682.029910] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 682.030289] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-69fdaa8d-fcea-4ec6-8581-accca03c7307 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 682.040987] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84a688fc-6ba0-4eb5-8b4e-a35d91762278 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 682.063139] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance cc41b8c4-c49a-4f23-bd16-ca2362a687d0 could not be found. [ 682.063373] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 682.063604] env[63593]: INFO nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Took 0.03 seconds to destroy the instance on the hypervisor. [ 682.063845] env[63593]: DEBUG oslo.service.loopingcall [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 682.064119] env[63593]: DEBUG nova.compute.manager [-] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 682.064190] env[63593]: DEBUG nova.network.neutron [-] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 682.095394] env[63593]: DEBUG nova.network.neutron [-] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 682.188894] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Releasing lock "refresh_cache-4869e3e4-48bb-446f-a15e-3a53bae1bc9b" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 682.189150] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 682.189325] env[63593]: DEBUG nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 682.189492] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 682.214927] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 682.427845] env[63593]: INFO nova.scheduler.client.report [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Deleted allocations for instance bb3a0a79-ef74-4fb0-a286-91b1b92df751 [ 682.488131] env[63593]: DEBUG nova.scheduler.client.report [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 682.598456] env[63593]: DEBUG nova.network.neutron [-] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 682.726623] env[63593]: DEBUG nova.network.neutron [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 682.938167] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9113c86d-f662-461a-a2c6-f92c8d01961c tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "bb3a0a79-ef74-4fb0-a286-91b1b92df751" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 62.336s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.995442] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.127s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.996117] env[63593]: ERROR nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ea563e31-7d0e-4720-b567-e7cef236d61f, please check neutron logs for more information. [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Traceback (most recent call last): [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self.driver.spawn(context, instance, image_meta, [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self._vmops.spawn(context, instance, image_meta, injected_files, [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] vm_ref = self.build_virtual_machine(instance, [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] vif_infos = vmwarevif.get_vif_info(self._session, [ 682.996117] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] for vif in network_info: [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] return self._sync_wrapper(fn, *args, **kwargs) [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self.wait() [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self[:] = self._gt.wait() [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] return self._exit_event.wait() [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] result = hub.switch() [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 682.996414] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] return self.greenlet.switch() [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] result = function(*args, **kwargs) [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] return func(*args, **kwargs) [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] raise e [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] nwinfo = self.network_api.allocate_for_instance( [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] created_port_ids = self._update_ports_for_instance( [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] with excutils.save_and_reraise_exception(): [ 682.996708] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] self.force_reraise() [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] raise self.value [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] updated_port = self._update_port( [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] _ensure_no_port_binding_failure(port) [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] raise exception.PortBindingFailed(port_id=port['id']) [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] nova.exception.PortBindingFailed: Binding failed for port ea563e31-7d0e-4720-b567-e7cef236d61f, please check neutron logs for more information. [ 682.997022] env[63593]: ERROR nova.compute.manager [instance: d46033b4-b3d5-494c-b25c-106475fe4409] [ 682.997292] env[63593]: DEBUG nova.compute.utils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Binding failed for port ea563e31-7d0e-4720-b567-e7cef236d61f, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 682.998834] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.889s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 683.001139] env[63593]: INFO nova.compute.claims [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 683.007373] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Build of instance d46033b4-b3d5-494c-b25c-106475fe4409 was re-scheduled: Binding failed for port ea563e31-7d0e-4720-b567-e7cef236d61f, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 683.007373] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 683.007373] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquiring lock "refresh_cache-d46033b4-b3d5-494c-b25c-106475fe4409" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 683.007373] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Acquired lock "refresh_cache-d46033b4-b3d5-494c-b25c-106475fe4409" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 683.007615] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 683.103213] env[63593]: INFO nova.compute.manager [-] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Took 1.04 seconds to deallocate network for instance. [ 683.104433] env[63593]: DEBUG nova.compute.claims [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 683.104433] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 683.229559] env[63593]: INFO nova.compute.manager [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] [instance: 4869e3e4-48bb-446f-a15e-3a53bae1bc9b] Took 1.04 seconds to deallocate network for instance. [ 683.441746] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 683.533398] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 683.607784] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 683.979368] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 684.114542] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Releasing lock "refresh_cache-d46033b4-b3d5-494c-b25c-106475fe4409" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 684.114806] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 684.114939] env[63593]: DEBUG nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 684.115117] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 684.146505] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 684.186730] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "a269ba6a-8439-448f-ad51-e3084c89c2a5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 684.186968] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "a269ba6a-8439-448f-ad51-e3084c89c2a5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 684.278318] env[63593]: INFO nova.scheduler.client.report [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Deleted allocations for instance 4869e3e4-48bb-446f-a15e-3a53bae1bc9b [ 684.562667] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e008700b-e0f4-4227-9113-e8049111ca3b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.569083] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b5fc181-ff77-45b5-9b90-12468e8653fc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.599087] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e40e693-48ba-4e8b-a461-489a8fa5263b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.607641] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef386736-1cff-4b0d-b6d6-1b36b1889130 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.622615] env[63593]: DEBUG nova.compute.provider_tree [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 684.648944] env[63593]: DEBUG nova.network.neutron [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 684.778572] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Acquiring lock "937998a3-dac3-4ce5-b363-7ceee313531a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 684.778809] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Lock "937998a3-dac3-4ce5-b363-7ceee313531a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 684.786131] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7aa1549a-486c-41d1-b1e5-142663cbd9f2 tempest-VolumesAssistedSnapshotsTest-251954128 tempest-VolumesAssistedSnapshotsTest-251954128-project-member] Lock "4869e3e4-48bb-446f-a15e-3a53bae1bc9b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 62.452s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.126893] env[63593]: DEBUG nova.scheduler.client.report [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 685.151441] env[63593]: INFO nova.compute.manager [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] [instance: d46033b4-b3d5-494c-b25c-106475fe4409] Took 1.04 seconds to deallocate network for instance. [ 685.289133] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 685.633534] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.635s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.634371] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 685.637733] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.593s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.640396] env[63593]: INFO nova.compute.claims [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 685.813682] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 686.144811] env[63593]: DEBUG nova.compute.utils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 686.148015] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 686.148196] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 686.180142] env[63593]: INFO nova.scheduler.client.report [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Deleted allocations for instance d46033b4-b3d5-494c-b25c-106475fe4409 [ 686.192644] env[63593]: DEBUG nova.policy [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4d38e618594c413f90c93b7dc077ab1e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8c5cb5838288429b8a33966196192ed5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 686.649548] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 686.687691] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b09d43f7-d697-417e-b7e9-af02d3dc8434 tempest-DeleteServersAdminTestJSON-1309299756 tempest-DeleteServersAdminTestJSON-1309299756-project-member] Lock "d46033b4-b3d5-494c-b25c-106475fe4409" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 63.059s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 686.730799] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Successfully created port: c1c4e6ed-f750-4e62-8634-bc833c43d305 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 687.115100] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0147a4c1-6f75-44e7-9c3a-4e3de1e6dfce {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.122150] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80bc3dac-4271-4a4a-a63f-d05af086b656 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.170670] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c14cfc82-8e27-480b-8664-7282ebfead5a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.180370] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07b40cbd-2cc9-4108-a966-c2e547842a4a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.195741] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 687.197928] env[63593]: DEBUG nova.compute.provider_tree [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 687.279552] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Acquiring lock "b5889795-9995-4211-8b3f-8d14680fde6f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 687.279552] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Lock "b5889795-9995-4211-8b3f-8d14680fde6f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 687.656047] env[63593]: DEBUG nova.compute.manager [req-622ddf43-c5f7-4a48-a819-ef549f1bafc8 req-0e7708ea-0822-4540-9d40-8b67a269ef14 service nova] [instance: c98de831-5d54-4a30-8414-9973b693c125] Received event network-changed-c1c4e6ed-f750-4e62-8634-bc833c43d305 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 687.656406] env[63593]: DEBUG nova.compute.manager [req-622ddf43-c5f7-4a48-a819-ef549f1bafc8 req-0e7708ea-0822-4540-9d40-8b67a269ef14 service nova] [instance: c98de831-5d54-4a30-8414-9973b693c125] Refreshing instance network info cache due to event network-changed-c1c4e6ed-f750-4e62-8634-bc833c43d305. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 687.656638] env[63593]: DEBUG oslo_concurrency.lockutils [req-622ddf43-c5f7-4a48-a819-ef549f1bafc8 req-0e7708ea-0822-4540-9d40-8b67a269ef14 service nova] Acquiring lock "refresh_cache-c98de831-5d54-4a30-8414-9973b693c125" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 687.656806] env[63593]: DEBUG oslo_concurrency.lockutils [req-622ddf43-c5f7-4a48-a819-ef549f1bafc8 req-0e7708ea-0822-4540-9d40-8b67a269ef14 service nova] Acquired lock "refresh_cache-c98de831-5d54-4a30-8414-9973b693c125" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 687.657444] env[63593]: DEBUG nova.network.neutron [req-622ddf43-c5f7-4a48-a819-ef549f1bafc8 req-0e7708ea-0822-4540-9d40-8b67a269ef14 service nova] [instance: c98de831-5d54-4a30-8414-9973b693c125] Refreshing network info cache for port c1c4e6ed-f750-4e62-8634-bc833c43d305 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 687.676897] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 687.705731] env[63593]: DEBUG nova.scheduler.client.report [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 687.720499] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 687.720499] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 687.720499] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 687.720671] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 687.720671] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 687.720671] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 687.720968] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 687.721168] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 687.721337] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 687.721824] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 687.721824] env[63593]: DEBUG nova.virt.hardware [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 687.722514] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5abac4b-af59-4039-9ac0-8e4790fa63da {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.726435] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 687.733035] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d864490-94ed-40f5-b128-9281e415733b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 687.850261] env[63593]: ERROR nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c1c4e6ed-f750-4e62-8634-bc833c43d305, please check neutron logs for more information. [ 687.850261] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 687.850261] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 687.850261] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 687.850261] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 687.850261] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 687.850261] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 687.850261] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 687.850261] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 687.850261] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 687.850261] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 687.850261] env[63593]: ERROR nova.compute.manager raise self.value [ 687.850261] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 687.850261] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 687.850261] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 687.850261] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 687.851441] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 687.851441] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 687.851441] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c1c4e6ed-f750-4e62-8634-bc833c43d305, please check neutron logs for more information. [ 687.851441] env[63593]: ERROR nova.compute.manager [ 687.851441] env[63593]: Traceback (most recent call last): [ 687.851441] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 687.851441] env[63593]: listener.cb(fileno) [ 687.851441] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 687.851441] env[63593]: result = function(*args, **kwargs) [ 687.851441] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 687.851441] env[63593]: return func(*args, **kwargs) [ 687.851441] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 687.851441] env[63593]: raise e [ 687.851441] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 687.851441] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 687.851441] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 687.851441] env[63593]: created_port_ids = self._update_ports_for_instance( [ 687.851441] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 687.851441] env[63593]: with excutils.save_and_reraise_exception(): [ 687.851441] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 687.851441] env[63593]: self.force_reraise() [ 687.851441] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 687.851441] env[63593]: raise self.value [ 687.851441] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 687.851441] env[63593]: updated_port = self._update_port( [ 687.851441] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 687.851441] env[63593]: _ensure_no_port_binding_failure(port) [ 687.851441] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 687.851441] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 687.852312] env[63593]: nova.exception.PortBindingFailed: Binding failed for port c1c4e6ed-f750-4e62-8634-bc833c43d305, please check neutron logs for more information. [ 687.852312] env[63593]: Removing descriptor: 21 [ 687.852312] env[63593]: ERROR nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c1c4e6ed-f750-4e62-8634-bc833c43d305, please check neutron logs for more information. [ 687.852312] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] Traceback (most recent call last): [ 687.852312] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 687.852312] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] yield resources [ 687.852312] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 687.852312] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self.driver.spawn(context, instance, image_meta, [ 687.852312] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 687.852312] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self._vmops.spawn(context, instance, image_meta, injected_files, [ 687.852312] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 687.852312] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] vm_ref = self.build_virtual_machine(instance, [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] vif_infos = vmwarevif.get_vif_info(self._session, [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] for vif in network_info: [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] return self._sync_wrapper(fn, *args, **kwargs) [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self.wait() [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self[:] = self._gt.wait() [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] return self._exit_event.wait() [ 687.852711] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] result = hub.switch() [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] return self.greenlet.switch() [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] result = function(*args, **kwargs) [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] return func(*args, **kwargs) [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] raise e [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] nwinfo = self.network_api.allocate_for_instance( [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 687.853012] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] created_port_ids = self._update_ports_for_instance( [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] with excutils.save_and_reraise_exception(): [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self.force_reraise() [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] raise self.value [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] updated_port = self._update_port( [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] _ensure_no_port_binding_failure(port) [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 687.853298] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] raise exception.PortBindingFailed(port_id=port['id']) [ 687.853551] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] nova.exception.PortBindingFailed: Binding failed for port c1c4e6ed-f750-4e62-8634-bc833c43d305, please check neutron logs for more information. [ 687.853551] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] [ 687.853551] env[63593]: INFO nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Terminating instance [ 687.853551] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Acquiring lock "refresh_cache-c98de831-5d54-4a30-8414-9973b693c125" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 688.178141] env[63593]: DEBUG nova.network.neutron [req-622ddf43-c5f7-4a48-a819-ef549f1bafc8 req-0e7708ea-0822-4540-9d40-8b67a269ef14 service nova] [instance: c98de831-5d54-4a30-8414-9973b693c125] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 688.223424] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.586s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 688.228200] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 688.239177] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.447s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 688.239615] env[63593]: INFO nova.compute.claims [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 688.291589] env[63593]: DEBUG nova.network.neutron [req-622ddf43-c5f7-4a48-a819-ef549f1bafc8 req-0e7708ea-0822-4540-9d40-8b67a269ef14 service nova] [instance: c98de831-5d54-4a30-8414-9973b693c125] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 688.744135] env[63593]: DEBUG nova.compute.utils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 688.748300] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 688.748466] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 688.793568] env[63593]: DEBUG oslo_concurrency.lockutils [req-622ddf43-c5f7-4a48-a819-ef549f1bafc8 req-0e7708ea-0822-4540-9d40-8b67a269ef14 service nova] Releasing lock "refresh_cache-c98de831-5d54-4a30-8414-9973b693c125" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 688.794777] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Acquired lock "refresh_cache-c98de831-5d54-4a30-8414-9973b693c125" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 688.795535] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 688.810569] env[63593]: DEBUG nova.policy [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '1db27961398346bb9912842b78a2ce98', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7acf74f1027c4081a3986a6e3e7d270e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 689.152884] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Successfully created port: d631597d-020c-4015-8354-c9fbbc90d136 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 689.251066] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 689.325041] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 689.464920] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 689.730035] env[63593]: DEBUG nova.compute.manager [req-f54ba552-5d3f-4ec5-a193-17935a5c24ce req-ec5afd7c-b1e7-4da8-86a9-81697b7895f7 service nova] [instance: c98de831-5d54-4a30-8414-9973b693c125] Received event network-vif-deleted-c1c4e6ed-f750-4e62-8634-bc833c43d305 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 689.734966] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d03cce1-f923-4264-9726-2db4b8565757 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.745716] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66ab0ce4-1b7d-423e-86fc-445eddcd8e63 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.784767] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd6abb5d-f7bd-44da-a138-eb20f6f1ebc8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.794252] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8d1b110-dd63-47d3-89f8-009947976a4a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.812632] env[63593]: DEBUG nova.compute.provider_tree [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 689.972890] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Releasing lock "refresh_cache-c98de831-5d54-4a30-8414-9973b693c125" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 689.973333] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 689.973518] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 689.973814] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-459261bc-6736-4116-8e1a-1e2cfa87ada3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.986064] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e47dfe5-750b-48b0-96e5-29722e5d09c9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.018551] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c98de831-5d54-4a30-8414-9973b693c125 could not be found. [ 690.018786] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 690.018961] env[63593]: INFO nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Took 0.05 seconds to destroy the instance on the hypervisor. [ 690.019225] env[63593]: DEBUG oslo.service.loopingcall [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 690.019440] env[63593]: DEBUG nova.compute.manager [-] [instance: c98de831-5d54-4a30-8414-9973b693c125] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 690.019533] env[63593]: DEBUG nova.network.neutron [-] [instance: c98de831-5d54-4a30-8414-9973b693c125] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 690.038679] env[63593]: DEBUG nova.network.neutron [-] [instance: c98de831-5d54-4a30-8414-9973b693c125] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 690.285718] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 690.298974] env[63593]: ERROR nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d631597d-020c-4015-8354-c9fbbc90d136, please check neutron logs for more information. [ 690.298974] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 690.298974] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 690.298974] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 690.298974] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 690.298974] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 690.298974] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 690.298974] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 690.298974] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 690.298974] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 690.298974] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 690.298974] env[63593]: ERROR nova.compute.manager raise self.value [ 690.298974] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 690.298974] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 690.298974] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 690.298974] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 690.299451] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 690.299451] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 690.299451] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d631597d-020c-4015-8354-c9fbbc90d136, please check neutron logs for more information. [ 690.299451] env[63593]: ERROR nova.compute.manager [ 690.299451] env[63593]: Traceback (most recent call last): [ 690.299451] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 690.299451] env[63593]: listener.cb(fileno) [ 690.299451] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 690.299451] env[63593]: result = function(*args, **kwargs) [ 690.299451] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 690.299451] env[63593]: return func(*args, **kwargs) [ 690.299451] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 690.299451] env[63593]: raise e [ 690.299451] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 690.299451] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 690.299451] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 690.299451] env[63593]: created_port_ids = self._update_ports_for_instance( [ 690.299451] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 690.299451] env[63593]: with excutils.save_and_reraise_exception(): [ 690.299451] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 690.299451] env[63593]: self.force_reraise() [ 690.299451] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 690.299451] env[63593]: raise self.value [ 690.299451] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 690.299451] env[63593]: updated_port = self._update_port( [ 690.299451] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 690.299451] env[63593]: _ensure_no_port_binding_failure(port) [ 690.299451] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 690.299451] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 690.300108] env[63593]: nova.exception.PortBindingFailed: Binding failed for port d631597d-020c-4015-8354-c9fbbc90d136, please check neutron logs for more information. [ 690.300108] env[63593]: Removing descriptor: 21 [ 690.314791] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 690.315039] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 690.315202] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 690.315376] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 690.315511] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 690.315650] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 690.315929] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 690.316173] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 690.316359] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 690.316568] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 690.316753] env[63593]: DEBUG nova.virt.hardware [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 690.317695] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af63b84e-b068-45fb-ba7f-814ad50e8578 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.321095] env[63593]: DEBUG nova.scheduler.client.report [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 690.332405] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc1ff7d0-56df-4392-9b9b-153658ff5fe0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.349585] env[63593]: ERROR nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d631597d-020c-4015-8354-c9fbbc90d136, please check neutron logs for more information. [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Traceback (most recent call last): [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] yield resources [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self.driver.spawn(context, instance, image_meta, [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self._vmops.spawn(context, instance, image_meta, injected_files, [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] vm_ref = self.build_virtual_machine(instance, [ 690.349585] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] vif_infos = vmwarevif.get_vif_info(self._session, [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] for vif in network_info: [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] return self._sync_wrapper(fn, *args, **kwargs) [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self.wait() [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self[:] = self._gt.wait() [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] return self._exit_event.wait() [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 690.349862] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] current.throw(*self._exc) [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] result = function(*args, **kwargs) [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] return func(*args, **kwargs) [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] raise e [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] nwinfo = self.network_api.allocate_for_instance( [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] created_port_ids = self._update_ports_for_instance( [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] with excutils.save_and_reraise_exception(): [ 690.350336] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self.force_reraise() [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] raise self.value [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] updated_port = self._update_port( [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] _ensure_no_port_binding_failure(port) [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] raise exception.PortBindingFailed(port_id=port['id']) [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] nova.exception.PortBindingFailed: Binding failed for port d631597d-020c-4015-8354-c9fbbc90d136, please check neutron logs for more information. [ 690.350706] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] [ 690.350706] env[63593]: INFO nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Terminating instance [ 690.353117] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "refresh_cache-9140518f-aae7-403d-acdd-8f8d80fc8552" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 690.353117] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquired lock "refresh_cache-9140518f-aae7-403d-acdd-8f8d80fc8552" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 690.353117] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 690.542733] env[63593]: DEBUG nova.network.neutron [-] [instance: c98de831-5d54-4a30-8414-9973b693c125] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 690.828081] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.590s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.828761] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 690.831464] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.733s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.832982] env[63593]: INFO nova.compute.claims [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 690.885980] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 691.045566] env[63593]: INFO nova.compute.manager [-] [instance: c98de831-5d54-4a30-8414-9973b693c125] Took 1.03 seconds to deallocate network for instance. [ 691.048053] env[63593]: DEBUG nova.compute.claims [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 691.048238] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.062786] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 691.338258] env[63593]: DEBUG nova.compute.utils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 691.342816] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 691.345751] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 691.432092] env[63593]: DEBUG nova.policy [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fd6454793bf142fd9ed5a2e01f8719db', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '627f52b1f1d4402881be4153c81674a0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 691.565682] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Releasing lock "refresh_cache-9140518f-aae7-403d-acdd-8f8d80fc8552" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 691.566130] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 691.566349] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 691.566650] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-129d5047-61e4-4f3e-bb20-45f9d6e20a4e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 691.577099] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f476baf4-7883-432a-be67-832033e81562 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 691.602794] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9140518f-aae7-403d-acdd-8f8d80fc8552 could not be found. [ 691.603029] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 691.603216] env[63593]: INFO nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Took 0.04 seconds to destroy the instance on the hypervisor. [ 691.603468] env[63593]: DEBUG oslo.service.loopingcall [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 691.603691] env[63593]: DEBUG nova.compute.manager [-] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 691.603783] env[63593]: DEBUG nova.network.neutron [-] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 691.630694] env[63593]: DEBUG nova.network.neutron [-] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 691.833202] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 691.834165] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 691.844248] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 691.860643] env[63593]: DEBUG nova.compute.manager [req-dbe9a706-f7f4-46c6-bb83-a660264637d9 req-0c65525e-2baa-41c3-9411-a838e59d253b service nova] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Received event network-changed-d631597d-020c-4015-8354-c9fbbc90d136 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 691.860951] env[63593]: DEBUG nova.compute.manager [req-dbe9a706-f7f4-46c6-bb83-a660264637d9 req-0c65525e-2baa-41c3-9411-a838e59d253b service nova] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Refreshing instance network info cache due to event network-changed-d631597d-020c-4015-8354-c9fbbc90d136. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 691.861263] env[63593]: DEBUG oslo_concurrency.lockutils [req-dbe9a706-f7f4-46c6-bb83-a660264637d9 req-0c65525e-2baa-41c3-9411-a838e59d253b service nova] Acquiring lock "refresh_cache-9140518f-aae7-403d-acdd-8f8d80fc8552" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 691.861498] env[63593]: DEBUG oslo_concurrency.lockutils [req-dbe9a706-f7f4-46c6-bb83-a660264637d9 req-0c65525e-2baa-41c3-9411-a838e59d253b service nova] Acquired lock "refresh_cache-9140518f-aae7-403d-acdd-8f8d80fc8552" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 691.861770] env[63593]: DEBUG nova.network.neutron [req-dbe9a706-f7f4-46c6-bb83-a660264637d9 req-0c65525e-2baa-41c3-9411-a838e59d253b service nova] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Refreshing network info cache for port d631597d-020c-4015-8354-c9fbbc90d136 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 692.131215] env[63593]: DEBUG nova.network.neutron [-] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 692.159538] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Successfully created port: c33cf0c6-f28e-40ce-8d43-7ce808cb6384 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 692.345458] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 692.345602] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 692.345717] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 692.396036] env[63593]: DEBUG nova.network.neutron [req-dbe9a706-f7f4-46c6-bb83-a660264637d9 req-0c65525e-2baa-41c3-9411-a838e59d253b service nova] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 692.405066] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e41ca507-c871-4b40-8cad-e6fad00780e4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.413257] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a07d58ef-efe2-4cbd-a8e6-17dc3322f584 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.448032] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f3a5a32-3741-4e9d-998a-a719f1340728 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.458434] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c24d537-9fce-40fc-8086-99052622fa32 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.472457] env[63593]: DEBUG nova.compute.provider_tree [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 692.525316] env[63593]: DEBUG nova.network.neutron [req-dbe9a706-f7f4-46c6-bb83-a660264637d9 req-0c65525e-2baa-41c3-9411-a838e59d253b service nova] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 692.639011] env[63593]: INFO nova.compute.manager [-] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Took 1.04 seconds to deallocate network for instance. [ 692.641482] env[63593]: DEBUG nova.compute.claims [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 692.641675] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 692.857617] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 692.857881] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 692.857881] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: c98de831-5d54-4a30-8414-9973b693c125] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 692.857992] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 692.858102] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 692.858228] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 692.858351] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 692.858556] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 692.859541] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 692.859541] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 692.859541] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 692.859541] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 692.859716] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 692.860395] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 692.860395] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 692.873449] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 692.902593] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 692.902834] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 692.902986] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 692.903179] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 692.903323] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 692.903466] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 692.903666] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 692.903821] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 692.903984] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 692.904168] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 692.904359] env[63593]: DEBUG nova.virt.hardware [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 692.905491] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c9439ca-c9a0-4890-8c5d-ff35b75c29ba {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.913713] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d21ee1c4-7f3b-4289-8f9d-a670a64e71b9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.979539] env[63593]: DEBUG nova.scheduler.client.report [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 693.028266] env[63593]: DEBUG oslo_concurrency.lockutils [req-dbe9a706-f7f4-46c6-bb83-a660264637d9 req-0c65525e-2baa-41c3-9411-a838e59d253b service nova] Releasing lock "refresh_cache-9140518f-aae7-403d-acdd-8f8d80fc8552" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 693.028549] env[63593]: DEBUG nova.compute.manager [req-dbe9a706-f7f4-46c6-bb83-a660264637d9 req-0c65525e-2baa-41c3-9411-a838e59d253b service nova] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Received event network-vif-deleted-d631597d-020c-4015-8354-c9fbbc90d136 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 693.363172] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 693.475938] env[63593]: ERROR nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384, please check neutron logs for more information. [ 693.475938] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 693.475938] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 693.475938] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 693.475938] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 693.475938] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 693.475938] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 693.475938] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 693.475938] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 693.475938] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 693.475938] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 693.475938] env[63593]: ERROR nova.compute.manager raise self.value [ 693.475938] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 693.475938] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 693.475938] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 693.475938] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 693.476406] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 693.476406] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 693.476406] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384, please check neutron logs for more information. [ 693.476406] env[63593]: ERROR nova.compute.manager [ 693.476406] env[63593]: Traceback (most recent call last): [ 693.476406] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 693.476406] env[63593]: listener.cb(fileno) [ 693.476406] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 693.476406] env[63593]: result = function(*args, **kwargs) [ 693.476406] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 693.476406] env[63593]: return func(*args, **kwargs) [ 693.476406] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 693.476406] env[63593]: raise e [ 693.476406] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 693.476406] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 693.476406] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 693.476406] env[63593]: created_port_ids = self._update_ports_for_instance( [ 693.476406] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 693.476406] env[63593]: with excutils.save_and_reraise_exception(): [ 693.476406] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 693.476406] env[63593]: self.force_reraise() [ 693.476406] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 693.476406] env[63593]: raise self.value [ 693.476406] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 693.476406] env[63593]: updated_port = self._update_port( [ 693.476406] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 693.476406] env[63593]: _ensure_no_port_binding_failure(port) [ 693.476406] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 693.476406] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 693.477203] env[63593]: nova.exception.PortBindingFailed: Binding failed for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384, please check neutron logs for more information. [ 693.477203] env[63593]: Removing descriptor: 21 [ 693.477203] env[63593]: ERROR nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384, please check neutron logs for more information. [ 693.477203] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Traceback (most recent call last): [ 693.477203] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 693.477203] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] yield resources [ 693.477203] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 693.477203] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self.driver.spawn(context, instance, image_meta, [ 693.477203] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 693.477203] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 693.477203] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 693.477203] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] vm_ref = self.build_virtual_machine(instance, [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] vif_infos = vmwarevif.get_vif_info(self._session, [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] for vif in network_info: [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] return self._sync_wrapper(fn, *args, **kwargs) [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self.wait() [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self[:] = self._gt.wait() [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] return self._exit_event.wait() [ 693.477594] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] result = hub.switch() [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] return self.greenlet.switch() [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] result = function(*args, **kwargs) [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] return func(*args, **kwargs) [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] raise e [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] nwinfo = self.network_api.allocate_for_instance( [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 693.477934] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] created_port_ids = self._update_ports_for_instance( [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] with excutils.save_and_reraise_exception(): [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self.force_reraise() [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] raise self.value [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] updated_port = self._update_port( [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] _ensure_no_port_binding_failure(port) [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 693.478485] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] raise exception.PortBindingFailed(port_id=port['id']) [ 693.478872] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] nova.exception.PortBindingFailed: Binding failed for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384, please check neutron logs for more information. [ 693.478872] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] [ 693.478872] env[63593]: INFO nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Terminating instance [ 693.479248] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Acquiring lock "refresh_cache-aa57c935-efea-4d95-9fda-65fa9106c90e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 693.479408] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Acquired lock "refresh_cache-aa57c935-efea-4d95-9fda-65fa9106c90e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 693.479571] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 693.485364] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.654s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 693.485892] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 693.488734] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.436s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 693.490758] env[63593]: INFO nova.compute.claims [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 693.898177] env[63593]: DEBUG nova.compute.manager [req-349e5206-0bc9-4116-93cb-f08f561a47f0 req-07f369ee-41ff-475f-86db-cb9661f2ab8a service nova] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Received event network-changed-c33cf0c6-f28e-40ce-8d43-7ce808cb6384 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 693.898451] env[63593]: DEBUG nova.compute.manager [req-349e5206-0bc9-4116-93cb-f08f561a47f0 req-07f369ee-41ff-475f-86db-cb9661f2ab8a service nova] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Refreshing instance network info cache due to event network-changed-c33cf0c6-f28e-40ce-8d43-7ce808cb6384. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 693.898735] env[63593]: DEBUG oslo_concurrency.lockutils [req-349e5206-0bc9-4116-93cb-f08f561a47f0 req-07f369ee-41ff-475f-86db-cb9661f2ab8a service nova] Acquiring lock "refresh_cache-aa57c935-efea-4d95-9fda-65fa9106c90e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 693.994951] env[63593]: DEBUG nova.compute.utils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 693.999346] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 693.999429] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 694.003032] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 694.054578] env[63593]: DEBUG nova.policy [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '263a83da7cd4471e9689e20c55812ccb', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9169fb1c5a2d4f4aa7f2ea07dda28996', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 694.105519] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 694.366119] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Successfully created port: c508017e-1190-44bd-bff3-ed58a549586d {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 694.500038] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 694.607897] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Releasing lock "refresh_cache-aa57c935-efea-4d95-9fda-65fa9106c90e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 694.608140] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 694.608336] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 694.611031] env[63593]: DEBUG oslo_concurrency.lockutils [req-349e5206-0bc9-4116-93cb-f08f561a47f0 req-07f369ee-41ff-475f-86db-cb9661f2ab8a service nova] Acquired lock "refresh_cache-aa57c935-efea-4d95-9fda-65fa9106c90e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 694.611031] env[63593]: DEBUG nova.network.neutron [req-349e5206-0bc9-4116-93cb-f08f561a47f0 req-07f369ee-41ff-475f-86db-cb9661f2ab8a service nova] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Refreshing network info cache for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 694.611933] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-342c46d4-acdb-4722-82b9-b82f1e4140e2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.621147] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4154ffb-9358-4ac2-8838-17a469192afb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.646149] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance aa57c935-efea-4d95-9fda-65fa9106c90e could not be found. [ 694.646507] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 694.646798] env[63593]: INFO nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 694.647409] env[63593]: DEBUG oslo.service.loopingcall [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 694.649259] env[63593]: DEBUG nova.compute.manager [-] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 694.649397] env[63593]: DEBUG nova.network.neutron [-] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 694.669966] env[63593]: DEBUG nova.network.neutron [-] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 694.949050] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe7082c4-e035-447b-b347-4414d12f0103 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.958103] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27d7a15c-64f6-4503-a1c3-d9ee90c18588 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.990111] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a86a120-ec23-457c-8ee2-dbdc06f7c92b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.997976] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25217a51-ee7f-4938-ab2a-a0d60b4a9685 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.581886] env[63593]: DEBUG nova.network.neutron [-] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 695.583390] env[63593]: DEBUG nova.compute.manager [req-75821cec-2025-4e03-8e8b-5df3b9426529 req-e93901da-8d5e-4606-bc25-f527ee4e0cef service nova] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Received event network-changed-c508017e-1190-44bd-bff3-ed58a549586d {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 695.583564] env[63593]: DEBUG nova.compute.manager [req-75821cec-2025-4e03-8e8b-5df3b9426529 req-e93901da-8d5e-4606-bc25-f527ee4e0cef service nova] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Refreshing instance network info cache due to event network-changed-c508017e-1190-44bd-bff3-ed58a549586d. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 695.583767] env[63593]: DEBUG oslo_concurrency.lockutils [req-75821cec-2025-4e03-8e8b-5df3b9426529 req-e93901da-8d5e-4606-bc25-f527ee4e0cef service nova] Acquiring lock "refresh_cache-6192c09e-64e7-4eb3-968e-922247892ca2" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 695.583904] env[63593]: DEBUG oslo_concurrency.lockutils [req-75821cec-2025-4e03-8e8b-5df3b9426529 req-e93901da-8d5e-4606-bc25-f527ee4e0cef service nova] Acquired lock "refresh_cache-6192c09e-64e7-4eb3-968e-922247892ca2" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 695.584064] env[63593]: DEBUG nova.network.neutron [req-75821cec-2025-4e03-8e8b-5df3b9426529 req-e93901da-8d5e-4606-bc25-f527ee4e0cef service nova] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Refreshing network info cache for port c508017e-1190-44bd-bff3-ed58a549586d {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 695.588033] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Acquiring lock "9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 695.588251] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Lock "9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 695.599457] env[63593]: DEBUG nova.compute.provider_tree [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 695.609940] env[63593]: DEBUG nova.network.neutron [req-349e5206-0bc9-4116-93cb-f08f561a47f0 req-07f369ee-41ff-475f-86db-cb9661f2ab8a service nova] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 695.690019] env[63593]: DEBUG nova.network.neutron [req-349e5206-0bc9-4116-93cb-f08f561a47f0 req-07f369ee-41ff-475f-86db-cb9661f2ab8a service nova] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 695.798548] env[63593]: ERROR nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c508017e-1190-44bd-bff3-ed58a549586d, please check neutron logs for more information. [ 695.798548] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 695.798548] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 695.798548] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 695.798548] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 695.798548] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 695.798548] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 695.798548] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 695.798548] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 695.798548] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 695.798548] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 695.798548] env[63593]: ERROR nova.compute.manager raise self.value [ 695.798548] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 695.798548] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 695.798548] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 695.798548] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 695.799110] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 695.799110] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 695.799110] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c508017e-1190-44bd-bff3-ed58a549586d, please check neutron logs for more information. [ 695.799110] env[63593]: ERROR nova.compute.manager [ 695.799110] env[63593]: Traceback (most recent call last): [ 695.799110] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 695.799110] env[63593]: listener.cb(fileno) [ 695.799110] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 695.799110] env[63593]: result = function(*args, **kwargs) [ 695.799110] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 695.799110] env[63593]: return func(*args, **kwargs) [ 695.799110] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 695.799110] env[63593]: raise e [ 695.799110] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 695.799110] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 695.799110] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 695.799110] env[63593]: created_port_ids = self._update_ports_for_instance( [ 695.799110] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 695.799110] env[63593]: with excutils.save_and_reraise_exception(): [ 695.799110] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 695.799110] env[63593]: self.force_reraise() [ 695.799110] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 695.799110] env[63593]: raise self.value [ 695.799110] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 695.799110] env[63593]: updated_port = self._update_port( [ 695.799110] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 695.799110] env[63593]: _ensure_no_port_binding_failure(port) [ 695.799110] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 695.799110] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 695.799959] env[63593]: nova.exception.PortBindingFailed: Binding failed for port c508017e-1190-44bd-bff3-ed58a549586d, please check neutron logs for more information. [ 695.799959] env[63593]: Removing descriptor: 19 [ 696.090628] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 696.095027] env[63593]: INFO nova.compute.manager [-] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Took 1.44 seconds to deallocate network for instance. [ 696.099203] env[63593]: DEBUG nova.compute.claims [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 696.099412] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.102260] env[63593]: DEBUG nova.scheduler.client.report [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 696.115395] env[63593]: DEBUG nova.network.neutron [req-75821cec-2025-4e03-8e8b-5df3b9426529 req-e93901da-8d5e-4606-bc25-f527ee4e0cef service nova] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 696.121015] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 696.121262] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 696.121416] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 696.121590] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 696.121751] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 696.121915] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 696.122079] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 696.122237] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 696.122410] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 696.122559] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 696.122725] env[63593]: DEBUG nova.virt.hardware [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 696.123661] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c5792f7-aac6-446f-b3e0-80f7cbb11fbd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.132340] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c46016ca-0188-4d13-aee6-9b812f213ef0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.146661] env[63593]: ERROR nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c508017e-1190-44bd-bff3-ed58a549586d, please check neutron logs for more information. [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Traceback (most recent call last): [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] yield resources [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self.driver.spawn(context, instance, image_meta, [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] vm_ref = self.build_virtual_machine(instance, [ 696.146661] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] vif_infos = vmwarevif.get_vif_info(self._session, [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] for vif in network_info: [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] return self._sync_wrapper(fn, *args, **kwargs) [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self.wait() [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self[:] = self._gt.wait() [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] return self._exit_event.wait() [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 696.147055] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] current.throw(*self._exc) [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] result = function(*args, **kwargs) [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] return func(*args, **kwargs) [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] raise e [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] nwinfo = self.network_api.allocate_for_instance( [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] created_port_ids = self._update_ports_for_instance( [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] with excutils.save_and_reraise_exception(): [ 696.147359] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self.force_reraise() [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] raise self.value [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] updated_port = self._update_port( [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] _ensure_no_port_binding_failure(port) [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] raise exception.PortBindingFailed(port_id=port['id']) [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] nova.exception.PortBindingFailed: Binding failed for port c508017e-1190-44bd-bff3-ed58a549586d, please check neutron logs for more information. [ 696.148424] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] [ 696.148424] env[63593]: INFO nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Terminating instance [ 696.148735] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Acquiring lock "refresh_cache-6192c09e-64e7-4eb3-968e-922247892ca2" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 696.192897] env[63593]: DEBUG oslo_concurrency.lockutils [req-349e5206-0bc9-4116-93cb-f08f561a47f0 req-07f369ee-41ff-475f-86db-cb9661f2ab8a service nova] Releasing lock "refresh_cache-aa57c935-efea-4d95-9fda-65fa9106c90e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 696.192897] env[63593]: DEBUG nova.compute.manager [req-349e5206-0bc9-4116-93cb-f08f561a47f0 req-07f369ee-41ff-475f-86db-cb9661f2ab8a service nova] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Received event network-vif-deleted-c33cf0c6-f28e-40ce-8d43-7ce808cb6384 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 696.195598] env[63593]: DEBUG nova.network.neutron [req-75821cec-2025-4e03-8e8b-5df3b9426529 req-e93901da-8d5e-4606-bc25-f527ee4e0cef service nova] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 696.609185] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.120s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 696.609788] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 696.612548] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.508s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 696.697834] env[63593]: DEBUG oslo_concurrency.lockutils [req-75821cec-2025-4e03-8e8b-5df3b9426529 req-e93901da-8d5e-4606-bc25-f527ee4e0cef service nova] Releasing lock "refresh_cache-6192c09e-64e7-4eb3-968e-922247892ca2" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 696.698262] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Acquired lock "refresh_cache-6192c09e-64e7-4eb3-968e-922247892ca2" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 696.698446] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 697.117291] env[63593]: DEBUG nova.compute.utils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 697.122483] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 697.123033] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 697.193605] env[63593]: DEBUG nova.policy [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a5dfbe43eff44535b32853c399877aa7', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '48721d7ae07d4cb69b3497892e3306c0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 697.220792] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 697.278946] env[63593]: DEBUG nova.compute.manager [req-0b9fae4a-edd0-44f5-b203-f894d6a3e30d req-1fc6c62a-f6fa-4446-847e-a1402a8bf6bb service nova] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Received event network-vif-deleted-c508017e-1190-44bd-bff3-ed58a549586d {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 697.317776] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 697.528453] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-165893d4-90b0-4f09-be6b-7491388551b7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.535837] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Successfully created port: e00622e0-1125-49ce-8821-524aceb7674e {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 697.538299] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4358911f-2d2e-46df-a489-35d12a9e4f06 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.567812] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b98a655-e893-4b2c-a3a8-3a028278e4cc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.575024] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fc6209f-35df-4e39-9b02-56979e789c83 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.588031] env[63593]: DEBUG nova.compute.provider_tree [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 697.624232] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 697.820788] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Releasing lock "refresh_cache-6192c09e-64e7-4eb3-968e-922247892ca2" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 697.821263] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 697.821461] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 697.821738] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1320adfc-60ed-4f81-8b9e-71bac85283c6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.830214] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3593fd25-c13d-4b3d-b5dd-8b31c4f8d99a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.852380] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6192c09e-64e7-4eb3-968e-922247892ca2 could not be found. [ 697.852593] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 697.852766] env[63593]: INFO nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Took 0.03 seconds to destroy the instance on the hypervisor. [ 697.852998] env[63593]: DEBUG oslo.service.loopingcall [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 697.853224] env[63593]: DEBUG nova.compute.manager [-] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 697.853318] env[63593]: DEBUG nova.network.neutron [-] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 697.867858] env[63593]: DEBUG nova.network.neutron [-] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 698.090909] env[63593]: DEBUG nova.scheduler.client.report [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 698.370146] env[63593]: DEBUG nova.network.neutron [-] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 698.468448] env[63593]: ERROR nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e00622e0-1125-49ce-8821-524aceb7674e, please check neutron logs for more information. [ 698.468448] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 698.468448] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 698.468448] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 698.468448] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 698.468448] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 698.468448] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 698.468448] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 698.468448] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 698.468448] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 698.468448] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 698.468448] env[63593]: ERROR nova.compute.manager raise self.value [ 698.468448] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 698.468448] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 698.468448] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 698.468448] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 698.468987] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 698.468987] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 698.468987] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e00622e0-1125-49ce-8821-524aceb7674e, please check neutron logs for more information. [ 698.468987] env[63593]: ERROR nova.compute.manager [ 698.468987] env[63593]: Traceback (most recent call last): [ 698.468987] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 698.468987] env[63593]: listener.cb(fileno) [ 698.468987] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 698.468987] env[63593]: result = function(*args, **kwargs) [ 698.468987] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 698.468987] env[63593]: return func(*args, **kwargs) [ 698.468987] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 698.468987] env[63593]: raise e [ 698.468987] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 698.468987] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 698.468987] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 698.468987] env[63593]: created_port_ids = self._update_ports_for_instance( [ 698.468987] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 698.468987] env[63593]: with excutils.save_and_reraise_exception(): [ 698.468987] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 698.468987] env[63593]: self.force_reraise() [ 698.468987] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 698.468987] env[63593]: raise self.value [ 698.468987] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 698.468987] env[63593]: updated_port = self._update_port( [ 698.468987] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 698.468987] env[63593]: _ensure_no_port_binding_failure(port) [ 698.468987] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 698.468987] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 698.469786] env[63593]: nova.exception.PortBindingFailed: Binding failed for port e00622e0-1125-49ce-8821-524aceb7674e, please check neutron logs for more information. [ 698.469786] env[63593]: Removing descriptor: 19 [ 698.596517] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.984s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 698.596871] env[63593]: ERROR nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 772db4f5-aac3-466d-a93a-3fa23280b635, please check neutron logs for more information. [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Traceback (most recent call last): [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self.driver.spawn(context, instance, image_meta, [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] vm_ref = self.build_virtual_machine(instance, [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] vif_infos = vmwarevif.get_vif_info(self._session, [ 698.596871] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] for vif in network_info: [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] return self._sync_wrapper(fn, *args, **kwargs) [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self.wait() [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self[:] = self._gt.wait() [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] return self._exit_event.wait() [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] result = hub.switch() [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 698.597251] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] return self.greenlet.switch() [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] result = function(*args, **kwargs) [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] return func(*args, **kwargs) [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] raise e [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] nwinfo = self.network_api.allocate_for_instance( [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] created_port_ids = self._update_ports_for_instance( [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] with excutils.save_and_reraise_exception(): [ 698.597536] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] self.force_reraise() [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] raise self.value [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] updated_port = self._update_port( [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] _ensure_no_port_binding_failure(port) [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] raise exception.PortBindingFailed(port_id=port['id']) [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] nova.exception.PortBindingFailed: Binding failed for port 772db4f5-aac3-466d-a93a-3fa23280b635, please check neutron logs for more information. [ 698.597816] env[63593]: ERROR nova.compute.manager [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] [ 698.598074] env[63593]: DEBUG nova.compute.utils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Binding failed for port 772db4f5-aac3-466d-a93a-3fa23280b635, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 698.598834] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.620s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 698.600764] env[63593]: INFO nova.compute.claims [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 698.603416] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Build of instance cc41b8c4-c49a-4f23-bd16-ca2362a687d0 was re-scheduled: Binding failed for port 772db4f5-aac3-466d-a93a-3fa23280b635, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 698.603836] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 698.604065] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "refresh_cache-cc41b8c4-c49a-4f23-bd16-ca2362a687d0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 698.604220] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquired lock "refresh_cache-cc41b8c4-c49a-4f23-bd16-ca2362a687d0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 698.604394] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 698.631426] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 698.656127] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 698.657112] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 698.657373] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 698.657597] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 698.657748] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 698.657894] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 698.658123] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 698.658287] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 698.658453] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 698.658613] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 698.658780] env[63593]: DEBUG nova.virt.hardware [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 698.659914] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a17d3e1f-77a5-43a2-91d7-c1d6ea17beef {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.667740] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-436bef0d-30fe-4784-89cd-d31572846e31 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.681741] env[63593]: ERROR nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e00622e0-1125-49ce-8821-524aceb7674e, please check neutron logs for more information. [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Traceback (most recent call last): [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] yield resources [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self.driver.spawn(context, instance, image_meta, [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self._vmops.spawn(context, instance, image_meta, injected_files, [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] vm_ref = self.build_virtual_machine(instance, [ 698.681741] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] vif_infos = vmwarevif.get_vif_info(self._session, [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] for vif in network_info: [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] return self._sync_wrapper(fn, *args, **kwargs) [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self.wait() [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self[:] = self._gt.wait() [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] return self._exit_event.wait() [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 698.682134] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] current.throw(*self._exc) [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] result = function(*args, **kwargs) [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] return func(*args, **kwargs) [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] raise e [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] nwinfo = self.network_api.allocate_for_instance( [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] created_port_ids = self._update_ports_for_instance( [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] with excutils.save_and_reraise_exception(): [ 698.682488] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self.force_reraise() [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] raise self.value [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] updated_port = self._update_port( [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] _ensure_no_port_binding_failure(port) [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] raise exception.PortBindingFailed(port_id=port['id']) [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] nova.exception.PortBindingFailed: Binding failed for port e00622e0-1125-49ce-8821-524aceb7674e, please check neutron logs for more information. [ 698.682836] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] [ 698.682836] env[63593]: INFO nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Terminating instance [ 698.683718] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Acquiring lock "refresh_cache-1205d83c-163f-41a4-ab32-a5a706763b33" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 698.683793] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Acquired lock "refresh_cache-1205d83c-163f-41a4-ab32-a5a706763b33" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 698.683956] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 698.872510] env[63593]: INFO nova.compute.manager [-] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Took 1.02 seconds to deallocate network for instance. [ 698.876625] env[63593]: DEBUG nova.compute.claims [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 698.876844] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 699.128441] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 699.204400] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 699.221932] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 699.284472] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 699.320469] env[63593]: DEBUG nova.compute.manager [req-e1d75ffa-af69-4744-a124-e2c6f934c489 req-4b055f92-9f97-4c4e-b3c2-19458b529de5 service nova] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Received event network-changed-e00622e0-1125-49ce-8821-524aceb7674e {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 699.320820] env[63593]: DEBUG nova.compute.manager [req-e1d75ffa-af69-4744-a124-e2c6f934c489 req-4b055f92-9f97-4c4e-b3c2-19458b529de5 service nova] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Refreshing instance network info cache due to event network-changed-e00622e0-1125-49ce-8821-524aceb7674e. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 699.321055] env[63593]: DEBUG oslo_concurrency.lockutils [req-e1d75ffa-af69-4744-a124-e2c6f934c489 req-4b055f92-9f97-4c4e-b3c2-19458b529de5 service nova] Acquiring lock "refresh_cache-1205d83c-163f-41a4-ab32-a5a706763b33" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 699.726443] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Releasing lock "refresh_cache-cc41b8c4-c49a-4f23-bd16-ca2362a687d0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 699.726702] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 699.726839] env[63593]: DEBUG nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 699.727948] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 699.740091] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 699.787819] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Releasing lock "refresh_cache-1205d83c-163f-41a4-ab32-a5a706763b33" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 699.788256] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 699.788452] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 699.790506] env[63593]: DEBUG oslo_concurrency.lockutils [req-e1d75ffa-af69-4744-a124-e2c6f934c489 req-4b055f92-9f97-4c4e-b3c2-19458b529de5 service nova] Acquired lock "refresh_cache-1205d83c-163f-41a4-ab32-a5a706763b33" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 699.790684] env[63593]: DEBUG nova.network.neutron [req-e1d75ffa-af69-4744-a124-e2c6f934c489 req-4b055f92-9f97-4c4e-b3c2-19458b529de5 service nova] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Refreshing network info cache for port e00622e0-1125-49ce-8821-524aceb7674e {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 699.791615] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b1af455f-85dc-464a-8f38-1ea9064840d2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.800777] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5726b76-5b94-4a12-8e65-206f5e832d12 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.824260] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1205d83c-163f-41a4-ab32-a5a706763b33 could not be found. [ 699.824479] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 699.824648] env[63593]: INFO nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Took 0.04 seconds to destroy the instance on the hypervisor. [ 699.824877] env[63593]: DEBUG oslo.service.loopingcall [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 699.826967] env[63593]: DEBUG nova.compute.manager [-] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 699.827075] env[63593]: DEBUG nova.network.neutron [-] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 699.842309] env[63593]: DEBUG nova.network.neutron [-] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 699.943632] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a215b068-f2b7-4ef0-832a-bbf4861554e9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.951019] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f05c43b0-7353-46f8-9155-e1dd6d6dae74 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.980146] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e32175a-bf65-4b7d-9870-64cbd47cd59f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.987176] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b23fe368-76f2-4807-aeaa-607a7007cd73 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 700.000661] env[63593]: DEBUG nova.compute.provider_tree [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 700.244153] env[63593]: DEBUG nova.network.neutron [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 700.309479] env[63593]: DEBUG nova.network.neutron [req-e1d75ffa-af69-4744-a124-e2c6f934c489 req-4b055f92-9f97-4c4e-b3c2-19458b529de5 service nova] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 700.344180] env[63593]: DEBUG nova.network.neutron [-] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 700.392082] env[63593]: DEBUG nova.network.neutron [req-e1d75ffa-af69-4744-a124-e2c6f934c489 req-4b055f92-9f97-4c4e-b3c2-19458b529de5 service nova] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 700.503995] env[63593]: DEBUG nova.scheduler.client.report [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 700.747750] env[63593]: INFO nova.compute.manager [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: cc41b8c4-c49a-4f23-bd16-ca2362a687d0] Took 1.02 seconds to deallocate network for instance. [ 700.846783] env[63593]: INFO nova.compute.manager [-] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Took 1.02 seconds to deallocate network for instance. [ 700.849047] env[63593]: DEBUG nova.compute.claims [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 700.849232] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 700.894207] env[63593]: DEBUG oslo_concurrency.lockutils [req-e1d75ffa-af69-4744-a124-e2c6f934c489 req-4b055f92-9f97-4c4e-b3c2-19458b529de5 service nova] Releasing lock "refresh_cache-1205d83c-163f-41a4-ab32-a5a706763b33" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 700.894455] env[63593]: DEBUG nova.compute.manager [req-e1d75ffa-af69-4744-a124-e2c6f934c489 req-4b055f92-9f97-4c4e-b3c2-19458b529de5 service nova] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Received event network-vif-deleted-e00622e0-1125-49ce-8821-524aceb7674e {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 701.009195] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.410s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 701.009710] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 701.012377] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.199s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 701.013814] env[63593]: INFO nova.compute.claims [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 701.518125] env[63593]: DEBUG nova.compute.utils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 701.521430] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 701.521599] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 701.573259] env[63593]: DEBUG nova.policy [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b52deea501b04fe4af5c6386645ba8b3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '105e0b4e28494bcb9b8048c6d74c7a60', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 701.777191] env[63593]: INFO nova.scheduler.client.report [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Deleted allocations for instance cc41b8c4-c49a-4f23-bd16-ca2362a687d0 [ 701.909335] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Successfully created port: 6a9869ab-f8a7-4643-9a58-8a69651e1bbc {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 702.022639] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 702.285499] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8bea9dc3-d7c2-43c5-9dac-757931341024 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "cc41b8c4-c49a-4f23-bd16-ca2362a687d0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 77.140s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 702.397875] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d369306-a4c8-4079-8ce6-3e292f8083ba {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.406432] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebf3e4b5-01ba-424a-a2fd-d06198885437 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.439704] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-930ef430-2563-42fb-a546-00134445ba4c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.447412] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c596cd2-d67f-4067-8d69-9ecad1924424 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.460997] env[63593]: DEBUG nova.compute.provider_tree [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 702.688489] env[63593]: DEBUG nova.compute.manager [req-7e5a841e-b215-4ff7-9009-550e05a9dad1 req-cc147e2c-821d-4a95-b6c1-6022ca1acfb8 service nova] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Received event network-changed-6a9869ab-f8a7-4643-9a58-8a69651e1bbc {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 702.688752] env[63593]: DEBUG nova.compute.manager [req-7e5a841e-b215-4ff7-9009-550e05a9dad1 req-cc147e2c-821d-4a95-b6c1-6022ca1acfb8 service nova] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Refreshing instance network info cache due to event network-changed-6a9869ab-f8a7-4643-9a58-8a69651e1bbc. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 702.688968] env[63593]: DEBUG oslo_concurrency.lockutils [req-7e5a841e-b215-4ff7-9009-550e05a9dad1 req-cc147e2c-821d-4a95-b6c1-6022ca1acfb8 service nova] Acquiring lock "refresh_cache-6bf23f6a-6727-448f-80d9-fe954addd49e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 702.689119] env[63593]: DEBUG oslo_concurrency.lockutils [req-7e5a841e-b215-4ff7-9009-550e05a9dad1 req-cc147e2c-821d-4a95-b6c1-6022ca1acfb8 service nova] Acquired lock "refresh_cache-6bf23f6a-6727-448f-80d9-fe954addd49e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 702.689284] env[63593]: DEBUG nova.network.neutron [req-7e5a841e-b215-4ff7-9009-550e05a9dad1 req-cc147e2c-821d-4a95-b6c1-6022ca1acfb8 service nova] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Refreshing network info cache for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 702.794628] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 702.963811] env[63593]: DEBUG nova.scheduler.client.report [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 703.024200] env[63593]: ERROR nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc, please check neutron logs for more information. [ 703.024200] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 703.024200] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 703.024200] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 703.024200] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 703.024200] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 703.024200] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 703.024200] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 703.024200] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 703.024200] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 703.024200] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 703.024200] env[63593]: ERROR nova.compute.manager raise self.value [ 703.024200] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 703.024200] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 703.024200] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 703.024200] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 703.024701] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 703.024701] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 703.024701] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc, please check neutron logs for more information. [ 703.024701] env[63593]: ERROR nova.compute.manager [ 703.024701] env[63593]: Traceback (most recent call last): [ 703.024701] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 703.024701] env[63593]: listener.cb(fileno) [ 703.024701] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 703.024701] env[63593]: result = function(*args, **kwargs) [ 703.024701] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 703.024701] env[63593]: return func(*args, **kwargs) [ 703.024701] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 703.024701] env[63593]: raise e [ 703.024701] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 703.024701] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 703.024701] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 703.024701] env[63593]: created_port_ids = self._update_ports_for_instance( [ 703.024701] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 703.024701] env[63593]: with excutils.save_and_reraise_exception(): [ 703.024701] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 703.024701] env[63593]: self.force_reraise() [ 703.024701] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 703.024701] env[63593]: raise self.value [ 703.024701] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 703.024701] env[63593]: updated_port = self._update_port( [ 703.024701] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 703.024701] env[63593]: _ensure_no_port_binding_failure(port) [ 703.024701] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 703.024701] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 703.025615] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc, please check neutron logs for more information. [ 703.025615] env[63593]: Removing descriptor: 21 [ 703.033607] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 703.059788] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 703.059993] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 703.060167] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 703.060351] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 703.060497] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 703.060641] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 703.060877] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 703.061609] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 703.061803] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 703.061972] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 703.062190] env[63593]: DEBUG nova.virt.hardware [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 703.063063] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-023a4d67-8fa5-484a-b370-7eb289b8ed36 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.072475] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04964787-1513-46ae-8075-a189726bb700 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.085843] env[63593]: ERROR nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc, please check neutron logs for more information. [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Traceback (most recent call last): [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] yield resources [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self.driver.spawn(context, instance, image_meta, [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] vm_ref = self.build_virtual_machine(instance, [ 703.085843] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] vif_infos = vmwarevif.get_vif_info(self._session, [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] for vif in network_info: [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] return self._sync_wrapper(fn, *args, **kwargs) [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self.wait() [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self[:] = self._gt.wait() [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] return self._exit_event.wait() [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 703.086229] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] current.throw(*self._exc) [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] result = function(*args, **kwargs) [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] return func(*args, **kwargs) [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] raise e [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] nwinfo = self.network_api.allocate_for_instance( [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] created_port_ids = self._update_ports_for_instance( [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] with excutils.save_and_reraise_exception(): [ 703.086594] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self.force_reraise() [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] raise self.value [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] updated_port = self._update_port( [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] _ensure_no_port_binding_failure(port) [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] raise exception.PortBindingFailed(port_id=port['id']) [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] nova.exception.PortBindingFailed: Binding failed for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc, please check neutron logs for more information. [ 703.086954] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] [ 703.086954] env[63593]: INFO nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Terminating instance [ 703.088420] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Acquiring lock "refresh_cache-6bf23f6a-6727-448f-80d9-fe954addd49e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 703.205804] env[63593]: DEBUG nova.network.neutron [req-7e5a841e-b215-4ff7-9009-550e05a9dad1 req-cc147e2c-821d-4a95-b6c1-6022ca1acfb8 service nova] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 703.287344] env[63593]: DEBUG nova.network.neutron [req-7e5a841e-b215-4ff7-9009-550e05a9dad1 req-cc147e2c-821d-4a95-b6c1-6022ca1acfb8 service nova] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 703.316545] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.469481] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.457s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.470052] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 703.472673] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.746s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.474088] env[63593]: INFO nova.compute.claims [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 703.789567] env[63593]: WARNING oslo_vmware.rw_handles [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 703.789567] env[63593]: ERROR oslo_vmware.rw_handles [ 703.789947] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/9c36615a-8b1a-4107-93e0-63eb7b1c5e27/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 703.792012] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 703.792280] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Copying Virtual Disk [datastore2] vmware_temp/9c36615a-8b1a-4107-93e0-63eb7b1c5e27/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/9c36615a-8b1a-4107-93e0-63eb7b1c5e27/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 703.792792] env[63593]: DEBUG oslo_concurrency.lockutils [req-7e5a841e-b215-4ff7-9009-550e05a9dad1 req-cc147e2c-821d-4a95-b6c1-6022ca1acfb8 service nova] Releasing lock "refresh_cache-6bf23f6a-6727-448f-80d9-fe954addd49e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 703.793147] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-757fd4d0-66b7-4541-bf28-9597eeea3604 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.795438] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Acquired lock "refresh_cache-6bf23f6a-6727-448f-80d9-fe954addd49e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 703.795683] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 703.803123] env[63593]: DEBUG oslo_vmware.api [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Waiting for the task: (returnval){ [ 703.803123] env[63593]: value = "task-1367956" [ 703.803123] env[63593]: _type = "Task" [ 703.803123] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 703.811785] env[63593]: DEBUG oslo_vmware.api [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Task: {'id': task-1367956, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 703.978482] env[63593]: DEBUG nova.compute.utils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 703.981855] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 703.982340] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 704.027175] env[63593]: DEBUG nova.policy [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3bb930b4d1534f3ea3abde038aa60100', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3101ac6769714328b1b7e2c5f371bccc', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 704.272041] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Successfully created port: a12681e7-678c-4636-a5b1-9d47074e6198 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 704.315303] env[63593]: DEBUG oslo_vmware.exceptions [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 704.316365] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 704.320020] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 704.320020] env[63593]: ERROR nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 704.320020] env[63593]: Faults: ['InvalidArgument'] [ 704.320020] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Traceback (most recent call last): [ 704.320020] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 704.320020] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] yield resources [ 704.320020] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 704.320020] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] self.driver.spawn(context, instance, image_meta, [ 704.320020] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 704.320020] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] self._fetch_image_if_missing(context, vi) [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] image_cache(vi, tmp_image_ds_loc) [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] vm_util.copy_virtual_disk( [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] session._wait_for_task(vmdk_copy_task) [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] return self.wait_for_task(task_ref) [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] return evt.wait() [ 704.320606] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] result = hub.switch() [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] return self.greenlet.switch() [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] self.f(*self.args, **self.kw) [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] raise exceptions.translate_fault(task_info.error) [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Faults: ['InvalidArgument'] [ 704.320948] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] [ 704.320948] env[63593]: INFO nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Terminating instance [ 704.321331] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquiring lock "refresh_cache-65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 704.321331] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquired lock "refresh_cache-65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 704.321331] env[63593]: DEBUG nova.network.neutron [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 704.415515] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 704.483242] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 704.714240] env[63593]: DEBUG nova.compute.manager [req-ccae5452-e0dd-4b98-ac60-22a86c89f786 req-b2699a09-3a35-4e1c-ac82-41bf0cf0cb56 service nova] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Received event network-vif-deleted-6a9869ab-f8a7-4643-9a58-8a69651e1bbc {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 704.846401] env[63593]: DEBUG nova.network.neutron [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 704.895963] env[63593]: DEBUG nova.network.neutron [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 704.898856] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-545c1972-35ad-40df-9d6d-cfeee70d5720 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.907066] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-459ac180-c553-4013-a239-539e2d7e3c6e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.936594] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Releasing lock "refresh_cache-6bf23f6a-6727-448f-80d9-fe954addd49e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 704.937013] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 704.937219] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 704.938030] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-de5bfc4d-34a0-407f-bf12-eba656d029c8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.940456] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc0521dc-b956-45be-b827-12dbc99ac239 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.951305] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e8ddcc6-00e1-4397-9dfc-f30bd83d05af {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.963156] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4321d34-c170-481d-8215-0be54d2dfa03 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.977670] env[63593]: DEBUG nova.compute.provider_tree [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 704.986994] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6bf23f6a-6727-448f-80d9-fe954addd49e could not be found. [ 704.986994] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 704.986994] env[63593]: INFO nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Took 0.05 seconds to destroy the instance on the hypervisor. [ 704.986994] env[63593]: DEBUG oslo.service.loopingcall [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 704.986994] env[63593]: DEBUG nova.compute.manager [-] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 704.987234] env[63593]: DEBUG nova.network.neutron [-] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 705.013336] env[63593]: DEBUG nova.network.neutron [-] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 705.318090] env[63593]: ERROR nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a12681e7-678c-4636-a5b1-9d47074e6198, please check neutron logs for more information. [ 705.318090] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 705.318090] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 705.318090] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 705.318090] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 705.318090] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 705.318090] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 705.318090] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 705.318090] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 705.318090] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 705.318090] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 705.318090] env[63593]: ERROR nova.compute.manager raise self.value [ 705.318090] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 705.318090] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 705.318090] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 705.318090] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 705.318574] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 705.318574] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 705.318574] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a12681e7-678c-4636-a5b1-9d47074e6198, please check neutron logs for more information. [ 705.318574] env[63593]: ERROR nova.compute.manager [ 705.318574] env[63593]: Traceback (most recent call last): [ 705.318574] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 705.318574] env[63593]: listener.cb(fileno) [ 705.318574] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 705.318574] env[63593]: result = function(*args, **kwargs) [ 705.318574] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 705.318574] env[63593]: return func(*args, **kwargs) [ 705.318574] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 705.318574] env[63593]: raise e [ 705.318574] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 705.318574] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 705.318574] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 705.318574] env[63593]: created_port_ids = self._update_ports_for_instance( [ 705.318574] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 705.318574] env[63593]: with excutils.save_and_reraise_exception(): [ 705.318574] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 705.318574] env[63593]: self.force_reraise() [ 705.318574] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 705.318574] env[63593]: raise self.value [ 705.318574] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 705.318574] env[63593]: updated_port = self._update_port( [ 705.318574] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 705.318574] env[63593]: _ensure_no_port_binding_failure(port) [ 705.318574] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 705.318574] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 705.319352] env[63593]: nova.exception.PortBindingFailed: Binding failed for port a12681e7-678c-4636-a5b1-9d47074e6198, please check neutron logs for more information. [ 705.319352] env[63593]: Removing descriptor: 21 [ 705.403138] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Releasing lock "refresh_cache-65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 705.403630] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 705.403760] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 705.404678] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee27adf0-b068-4460-86e2-bffe343e52de {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 705.414013] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 705.414268] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-00fdf79e-8352-41ec-9b34-8bc9e7118f37 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 705.449514] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 705.449789] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 705.449982] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Deleting the datastore file [datastore2] 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 705.450262] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-a5581399-c7ac-47ec-84fe-2f5184f99573 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 705.458954] env[63593]: DEBUG oslo_vmware.api [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Waiting for the task: (returnval){ [ 705.458954] env[63593]: value = "task-1367958" [ 705.458954] env[63593]: _type = "Task" [ 705.458954] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 705.466090] env[63593]: DEBUG oslo_vmware.api [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Task: {'id': task-1367958, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 705.488342] env[63593]: DEBUG nova.scheduler.client.report [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 705.494350] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 705.517966] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 705.518221] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 705.518374] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 705.518548] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 705.518689] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 705.518830] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 705.519036] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 705.519196] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 705.519355] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 705.519514] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 705.519678] env[63593]: DEBUG nova.virt.hardware [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 705.519959] env[63593]: DEBUG nova.network.neutron [-] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 705.521481] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2550290a-1831-45ea-93a8-d9580e51bbf8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 705.529626] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f2ea46a-861d-4f0d-8b00-dcbb191dae9c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 705.544129] env[63593]: ERROR nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a12681e7-678c-4636-a5b1-9d47074e6198, please check neutron logs for more information. [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Traceback (most recent call last): [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] yield resources [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self.driver.spawn(context, instance, image_meta, [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self._vmops.spawn(context, instance, image_meta, injected_files, [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] vm_ref = self.build_virtual_machine(instance, [ 705.544129] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] vif_infos = vmwarevif.get_vif_info(self._session, [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] for vif in network_info: [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] return self._sync_wrapper(fn, *args, **kwargs) [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self.wait() [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self[:] = self._gt.wait() [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] return self._exit_event.wait() [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 705.544574] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] current.throw(*self._exc) [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] result = function(*args, **kwargs) [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] return func(*args, **kwargs) [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] raise e [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] nwinfo = self.network_api.allocate_for_instance( [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] created_port_ids = self._update_ports_for_instance( [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] with excutils.save_and_reraise_exception(): [ 705.544858] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self.force_reraise() [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] raise self.value [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] updated_port = self._update_port( [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] _ensure_no_port_binding_failure(port) [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] raise exception.PortBindingFailed(port_id=port['id']) [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] nova.exception.PortBindingFailed: Binding failed for port a12681e7-678c-4636-a5b1-9d47074e6198, please check neutron logs for more information. [ 705.545173] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] [ 705.545173] env[63593]: INFO nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Terminating instance [ 705.546221] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "refresh_cache-6462167e-ae83-4d73-bfb2-ec7024e56a07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 705.546403] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "refresh_cache-6462167e-ae83-4d73-bfb2-ec7024e56a07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 705.546573] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 705.967764] env[63593]: DEBUG oslo_vmware.api [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Task: {'id': task-1367958, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.032666} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 705.968074] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 705.968299] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 705.968477] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 705.968654] env[63593]: INFO nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Took 0.56 seconds to destroy the instance on the hypervisor. [ 705.968881] env[63593]: DEBUG oslo.service.loopingcall [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 705.969086] env[63593]: DEBUG nova.compute.manager [-] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 705.971197] env[63593]: DEBUG nova.compute.claims [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 705.971364] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 705.993245] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.521s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 705.993722] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 705.996151] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.948s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 706.025617] env[63593]: INFO nova.compute.manager [-] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Took 1.04 seconds to deallocate network for instance. [ 706.027938] env[63593]: DEBUG nova.compute.claims [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 706.028122] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 706.063274] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 706.153589] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 706.500949] env[63593]: DEBUG nova.compute.utils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 706.506066] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 706.506066] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 706.559660] env[63593]: DEBUG nova.policy [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '33be5e80c65444eaaf2e78b4ee5e497a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cfe14210e5b1405d85ade6e6d022b215', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 706.656750] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "refresh_cache-6462167e-ae83-4d73-bfb2-ec7024e56a07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 706.657274] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 706.657510] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 706.657836] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cfbc80c8-ec5d-49c4-94ad-7dcac5e2a2ae {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.668348] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce91517f-1504-4101-afa2-b0f247276641 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.693370] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6462167e-ae83-4d73-bfb2-ec7024e56a07 could not be found. [ 706.693754] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 706.693989] env[63593]: INFO nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Took 0.04 seconds to destroy the instance on the hypervisor. [ 706.695033] env[63593]: DEBUG oslo.service.loopingcall [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 706.699141] env[63593]: DEBUG nova.compute.manager [-] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 706.699141] env[63593]: DEBUG nova.network.neutron [-] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 706.723162] env[63593]: DEBUG nova.network.neutron [-] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 706.772395] env[63593]: DEBUG nova.compute.manager [req-72d31cee-21cc-4470-af45-87ee2e2c211c req-5939fb7d-06a1-49e3-afeb-960fc621c26a service nova] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Received event network-changed-a12681e7-678c-4636-a5b1-9d47074e6198 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 706.772596] env[63593]: DEBUG nova.compute.manager [req-72d31cee-21cc-4470-af45-87ee2e2c211c req-5939fb7d-06a1-49e3-afeb-960fc621c26a service nova] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Refreshing instance network info cache due to event network-changed-a12681e7-678c-4636-a5b1-9d47074e6198. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 706.773040] env[63593]: DEBUG oslo_concurrency.lockutils [req-72d31cee-21cc-4470-af45-87ee2e2c211c req-5939fb7d-06a1-49e3-afeb-960fc621c26a service nova] Acquiring lock "refresh_cache-6462167e-ae83-4d73-bfb2-ec7024e56a07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 706.773040] env[63593]: DEBUG oslo_concurrency.lockutils [req-72d31cee-21cc-4470-af45-87ee2e2c211c req-5939fb7d-06a1-49e3-afeb-960fc621c26a service nova] Acquired lock "refresh_cache-6462167e-ae83-4d73-bfb2-ec7024e56a07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 706.773150] env[63593]: DEBUG nova.network.neutron [req-72d31cee-21cc-4470-af45-87ee2e2c211c req-5939fb7d-06a1-49e3-afeb-960fc621c26a service nova] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Refreshing network info cache for port a12681e7-678c-4636-a5b1-9d47074e6198 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 706.919919] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c1c3313-f0ad-4f74-8601-74e997a34038 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.923215] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Successfully created port: 1b2017a6-6125-4b40-9e28-1d038ee03a57 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 706.931521] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9f6a971-e81d-4ddf-a8ff-2fa5a653f714 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.964202] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67d83709-5c2f-4971-9413-435f40aaceff {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.972584] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ac6ff86-605e-4076-94d5-a26b42989ade {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.986924] env[63593]: DEBUG nova.compute.provider_tree [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 707.008259] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 707.227579] env[63593]: DEBUG nova.network.neutron [-] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 707.300713] env[63593]: DEBUG nova.network.neutron [req-72d31cee-21cc-4470-af45-87ee2e2c211c req-5939fb7d-06a1-49e3-afeb-960fc621c26a service nova] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 707.417792] env[63593]: DEBUG nova.network.neutron [req-72d31cee-21cc-4470-af45-87ee2e2c211c req-5939fb7d-06a1-49e3-afeb-960fc621c26a service nova] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 707.490378] env[63593]: DEBUG nova.scheduler.client.report [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 707.731191] env[63593]: INFO nova.compute.manager [-] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Took 1.03 seconds to deallocate network for instance. [ 707.734257] env[63593]: DEBUG nova.compute.claims [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 707.734577] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 707.922128] env[63593]: DEBUG oslo_concurrency.lockutils [req-72d31cee-21cc-4470-af45-87ee2e2c211c req-5939fb7d-06a1-49e3-afeb-960fc621c26a service nova] Releasing lock "refresh_cache-6462167e-ae83-4d73-bfb2-ec7024e56a07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 707.922374] env[63593]: DEBUG nova.compute.manager [req-72d31cee-21cc-4470-af45-87ee2e2c211c req-5939fb7d-06a1-49e3-afeb-960fc621c26a service nova] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Received event network-vif-deleted-a12681e7-678c-4636-a5b1-9d47074e6198 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 707.978109] env[63593]: ERROR nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1b2017a6-6125-4b40-9e28-1d038ee03a57, please check neutron logs for more information. [ 707.978109] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 707.978109] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 707.978109] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 707.978109] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 707.978109] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 707.978109] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 707.978109] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 707.978109] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 707.978109] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 707.978109] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 707.978109] env[63593]: ERROR nova.compute.manager raise self.value [ 707.978109] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 707.978109] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 707.978109] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 707.978109] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 707.978780] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 707.978780] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 707.978780] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1b2017a6-6125-4b40-9e28-1d038ee03a57, please check neutron logs for more information. [ 707.978780] env[63593]: ERROR nova.compute.manager [ 707.978780] env[63593]: Traceback (most recent call last): [ 707.978780] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 707.978780] env[63593]: listener.cb(fileno) [ 707.978780] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 707.978780] env[63593]: result = function(*args, **kwargs) [ 707.978780] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 707.978780] env[63593]: return func(*args, **kwargs) [ 707.978780] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 707.978780] env[63593]: raise e [ 707.978780] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 707.978780] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 707.978780] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 707.978780] env[63593]: created_port_ids = self._update_ports_for_instance( [ 707.978780] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 707.978780] env[63593]: with excutils.save_and_reraise_exception(): [ 707.978780] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 707.978780] env[63593]: self.force_reraise() [ 707.978780] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 707.978780] env[63593]: raise self.value [ 707.978780] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 707.978780] env[63593]: updated_port = self._update_port( [ 707.978780] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 707.978780] env[63593]: _ensure_no_port_binding_failure(port) [ 707.978780] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 707.978780] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 707.979600] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 1b2017a6-6125-4b40-9e28-1d038ee03a57, please check neutron logs for more information. [ 707.979600] env[63593]: Removing descriptor: 21 [ 707.999480] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.002s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 707.999881] env[63593]: ERROR nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c1c4e6ed-f750-4e62-8634-bc833c43d305, please check neutron logs for more information. [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] Traceback (most recent call last): [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self.driver.spawn(context, instance, image_meta, [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self._vmops.spawn(context, instance, image_meta, injected_files, [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] vm_ref = self.build_virtual_machine(instance, [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] vif_infos = vmwarevif.get_vif_info(self._session, [ 707.999881] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] for vif in network_info: [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] return self._sync_wrapper(fn, *args, **kwargs) [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self.wait() [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self[:] = self._gt.wait() [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] return self._exit_event.wait() [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] result = hub.switch() [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 708.000481] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] return self.greenlet.switch() [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] result = function(*args, **kwargs) [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] return func(*args, **kwargs) [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] raise e [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] nwinfo = self.network_api.allocate_for_instance( [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] created_port_ids = self._update_ports_for_instance( [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] with excutils.save_and_reraise_exception(): [ 708.001203] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] self.force_reraise() [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] raise self.value [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] updated_port = self._update_port( [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] _ensure_no_port_binding_failure(port) [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] raise exception.PortBindingFailed(port_id=port['id']) [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] nova.exception.PortBindingFailed: Binding failed for port c1c4e6ed-f750-4e62-8634-bc833c43d305, please check neutron logs for more information. [ 708.001769] env[63593]: ERROR nova.compute.manager [instance: c98de831-5d54-4a30-8414-9973b693c125] [ 708.002257] env[63593]: DEBUG nova.compute.utils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Binding failed for port c1c4e6ed-f750-4e62-8634-bc833c43d305, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 708.002675] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.361s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 708.006865] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Build of instance c98de831-5d54-4a30-8414-9973b693c125 was re-scheduled: Binding failed for port c1c4e6ed-f750-4e62-8634-bc833c43d305, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 708.007403] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 708.007789] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Acquiring lock "refresh_cache-c98de831-5d54-4a30-8414-9973b693c125" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 708.008012] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Acquired lock "refresh_cache-c98de831-5d54-4a30-8414-9973b693c125" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 708.008247] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 708.017016] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 708.045113] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 708.045369] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 708.045522] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 708.045695] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 708.045837] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 708.045978] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 708.046205] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 708.046386] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 708.046558] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 708.046718] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 708.046887] env[63593]: DEBUG nova.virt.hardware [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 708.048011] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87deee33-84a1-4e68-bbe9-509c9437e090 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.055944] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb92c7b2-ba8e-4963-999e-48fddd1edcc9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.069804] env[63593]: ERROR nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1b2017a6-6125-4b40-9e28-1d038ee03a57, please check neutron logs for more information. [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Traceback (most recent call last): [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] yield resources [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self.driver.spawn(context, instance, image_meta, [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self._vmops.spawn(context, instance, image_meta, injected_files, [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] vm_ref = self.build_virtual_machine(instance, [ 708.069804] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] vif_infos = vmwarevif.get_vif_info(self._session, [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] for vif in network_info: [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] return self._sync_wrapper(fn, *args, **kwargs) [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self.wait() [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self[:] = self._gt.wait() [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] return self._exit_event.wait() [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 708.070383] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] current.throw(*self._exc) [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] result = function(*args, **kwargs) [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] return func(*args, **kwargs) [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] raise e [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] nwinfo = self.network_api.allocate_for_instance( [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] created_port_ids = self._update_ports_for_instance( [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] with excutils.save_and_reraise_exception(): [ 708.070723] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self.force_reraise() [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] raise self.value [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] updated_port = self._update_port( [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] _ensure_no_port_binding_failure(port) [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] raise exception.PortBindingFailed(port_id=port['id']) [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] nova.exception.PortBindingFailed: Binding failed for port 1b2017a6-6125-4b40-9e28-1d038ee03a57, please check neutron logs for more information. [ 708.071079] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] [ 708.071079] env[63593]: INFO nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Terminating instance [ 708.072111] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "refresh_cache-04de495d-8896-4c6d-a059-9a80477e3b07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 708.072270] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquired lock "refresh_cache-04de495d-8896-4c6d-a059-9a80477e3b07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 708.072432] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 708.524417] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 708.593215] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 708.614877] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 708.691603] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 708.805021] env[63593]: DEBUG nova.compute.manager [req-90fe9a10-2413-4779-b265-bf52bf32d1fd req-047b5d9b-36c8-44e1-a19e-9f1f1a4357c4 service nova] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Received event network-changed-1b2017a6-6125-4b40-9e28-1d038ee03a57 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 708.805335] env[63593]: DEBUG nova.compute.manager [req-90fe9a10-2413-4779-b265-bf52bf32d1fd req-047b5d9b-36c8-44e1-a19e-9f1f1a4357c4 service nova] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Refreshing instance network info cache due to event network-changed-1b2017a6-6125-4b40-9e28-1d038ee03a57. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 708.805462] env[63593]: DEBUG oslo_concurrency.lockutils [req-90fe9a10-2413-4779-b265-bf52bf32d1fd req-047b5d9b-36c8-44e1-a19e-9f1f1a4357c4 service nova] Acquiring lock "refresh_cache-04de495d-8896-4c6d-a059-9a80477e3b07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 708.870841] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97bf76b3-49ec-4be6-bd69-ab35aabb523f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.878287] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa473519-2230-4843-a3db-3af169f39a47 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.909532] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-151c75a6-2d7b-4444-bdaf-395e1cadb224 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.917103] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf449107-bbc6-4384-bc27-53645a534593 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 708.929990] env[63593]: DEBUG nova.compute.provider_tree [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 709.120047] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Releasing lock "refresh_cache-c98de831-5d54-4a30-8414-9973b693c125" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 709.120047] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 709.120047] env[63593]: DEBUG nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 709.120412] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 709.134578] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 709.193280] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Releasing lock "refresh_cache-04de495d-8896-4c6d-a059-9a80477e3b07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 709.193715] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 709.193909] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 709.194335] env[63593]: DEBUG oslo_concurrency.lockutils [req-90fe9a10-2413-4779-b265-bf52bf32d1fd req-047b5d9b-36c8-44e1-a19e-9f1f1a4357c4 service nova] Acquired lock "refresh_cache-04de495d-8896-4c6d-a059-9a80477e3b07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 709.194608] env[63593]: DEBUG nova.network.neutron [req-90fe9a10-2413-4779-b265-bf52bf32d1fd req-047b5d9b-36c8-44e1-a19e-9f1f1a4357c4 service nova] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Refreshing network info cache for port 1b2017a6-6125-4b40-9e28-1d038ee03a57 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 709.195648] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f12b3623-e607-47ba-888e-97b5de9880a8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.204856] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a6c7937-c416-4118-af3f-4e23dba36992 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.226607] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 04de495d-8896-4c6d-a059-9a80477e3b07 could not be found. [ 709.226830] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 709.227012] env[63593]: INFO nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Took 0.03 seconds to destroy the instance on the hypervisor. [ 709.227253] env[63593]: DEBUG oslo.service.loopingcall [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 709.227485] env[63593]: DEBUG nova.compute.manager [-] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 709.227587] env[63593]: DEBUG nova.network.neutron [-] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 709.241484] env[63593]: DEBUG nova.network.neutron [-] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 709.432886] env[63593]: DEBUG nova.scheduler.client.report [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 709.637078] env[63593]: DEBUG nova.network.neutron [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 709.715209] env[63593]: DEBUG nova.network.neutron [req-90fe9a10-2413-4779-b265-bf52bf32d1fd req-047b5d9b-36c8-44e1-a19e-9f1f1a4357c4 service nova] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 709.743525] env[63593]: DEBUG nova.network.neutron [-] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 709.807786] env[63593]: DEBUG nova.network.neutron [req-90fe9a10-2413-4779-b265-bf52bf32d1fd req-047b5d9b-36c8-44e1-a19e-9f1f1a4357c4 service nova] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 709.941044] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.938s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.941044] env[63593]: ERROR nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d631597d-020c-4015-8354-c9fbbc90d136, please check neutron logs for more information. [ 709.941044] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Traceback (most recent call last): [ 709.941044] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 709.941044] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self.driver.spawn(context, instance, image_meta, [ 709.941044] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 709.941044] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self._vmops.spawn(context, instance, image_meta, injected_files, [ 709.941044] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 709.941044] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] vm_ref = self.build_virtual_machine(instance, [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] vif_infos = vmwarevif.get_vif_info(self._session, [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] for vif in network_info: [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] return self._sync_wrapper(fn, *args, **kwargs) [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self.wait() [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self[:] = self._gt.wait() [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] return self._exit_event.wait() [ 709.941378] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] current.throw(*self._exc) [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] result = function(*args, **kwargs) [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] return func(*args, **kwargs) [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] raise e [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] nwinfo = self.network_api.allocate_for_instance( [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] created_port_ids = self._update_ports_for_instance( [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 709.941738] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] with excutils.save_and_reraise_exception(): [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] self.force_reraise() [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] raise self.value [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] updated_port = self._update_port( [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] _ensure_no_port_binding_failure(port) [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] raise exception.PortBindingFailed(port_id=port['id']) [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] nova.exception.PortBindingFailed: Binding failed for port d631597d-020c-4015-8354-c9fbbc90d136, please check neutron logs for more information. [ 709.942095] env[63593]: ERROR nova.compute.manager [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] [ 709.942423] env[63593]: DEBUG nova.compute.utils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Binding failed for port d631597d-020c-4015-8354-c9fbbc90d136, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 709.943438] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 16.580s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.943543] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.943673] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 709.944287] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.845s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.947494] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Build of instance 9140518f-aae7-403d-acdd-8f8d80fc8552 was re-scheduled: Binding failed for port d631597d-020c-4015-8354-c9fbbc90d136, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 709.947947] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 709.948195] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquiring lock "refresh_cache-9140518f-aae7-403d-acdd-8f8d80fc8552" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 709.948341] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Acquired lock "refresh_cache-9140518f-aae7-403d-acdd-8f8d80fc8552" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 709.948499] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 709.951027] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98a8c6e6-74b2-4358-ba33-f44dfd9a9605 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.960164] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acb68f80-4a3d-4412-83f4-108966a1283e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.974807] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d6d6ed6-0f49-4d85-b610-c7fbbc9bab61 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.981242] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8b45c96-edaa-4447-ab88-1b19fc86d42b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.013655] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181421MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 710.014514] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.141072] env[63593]: INFO nova.compute.manager [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] [instance: c98de831-5d54-4a30-8414-9973b693c125] Took 1.02 seconds to deallocate network for instance. [ 710.246521] env[63593]: INFO nova.compute.manager [-] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Took 1.02 seconds to deallocate network for instance. [ 710.248805] env[63593]: DEBUG nova.compute.claims [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 710.248989] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.310315] env[63593]: DEBUG oslo_concurrency.lockutils [req-90fe9a10-2413-4779-b265-bf52bf32d1fd req-047b5d9b-36c8-44e1-a19e-9f1f1a4357c4 service nova] Releasing lock "refresh_cache-04de495d-8896-4c6d-a059-9a80477e3b07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 710.310789] env[63593]: DEBUG nova.compute.manager [req-90fe9a10-2413-4779-b265-bf52bf32d1fd req-047b5d9b-36c8-44e1-a19e-9f1f1a4357c4 service nova] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Received event network-vif-deleted-1b2017a6-6125-4b40-9e28-1d038ee03a57 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 710.472653] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 710.554168] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 710.803516] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4031b66-7a7c-4996-b72f-2685f0fe0b78 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.811935] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d34c0f2-ef1a-4a15-8aff-43f7b564d2c2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.845329] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ce0309b-f286-4f68-87b4-17d4733ba3b6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.852779] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ab0ac36-cbe4-456e-b975-2e9ffa46ff13 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.866539] env[63593]: DEBUG nova.compute.provider_tree [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 711.056578] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Releasing lock "refresh_cache-9140518f-aae7-403d-acdd-8f8d80fc8552" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 711.056859] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 711.057068] env[63593]: DEBUG nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 711.057244] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 711.079602] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 711.171836] env[63593]: INFO nova.scheduler.client.report [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Deleted allocations for instance c98de831-5d54-4a30-8414-9973b693c125 [ 711.371257] env[63593]: DEBUG nova.scheduler.client.report [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 711.584618] env[63593]: DEBUG nova.network.neutron [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 711.680473] env[63593]: DEBUG oslo_concurrency.lockutils [None req-63b9a6ab-cfac-400c-8738-b8382b837e24 tempest-ServersTestManualDisk-1944496274 tempest-ServersTestManualDisk-1944496274-project-member] Lock "c98de831-5d54-4a30-8414-9973b693c125" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 85.555s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 711.876988] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.932s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 711.877287] env[63593]: ERROR nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384, please check neutron logs for more information. [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Traceback (most recent call last): [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self.driver.spawn(context, instance, image_meta, [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] vm_ref = self.build_virtual_machine(instance, [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] vif_infos = vmwarevif.get_vif_info(self._session, [ 711.877287] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] for vif in network_info: [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] return self._sync_wrapper(fn, *args, **kwargs) [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self.wait() [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self[:] = self._gt.wait() [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] return self._exit_event.wait() [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] result = hub.switch() [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 711.877705] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] return self.greenlet.switch() [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] result = function(*args, **kwargs) [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] return func(*args, **kwargs) [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] raise e [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] nwinfo = self.network_api.allocate_for_instance( [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] created_port_ids = self._update_ports_for_instance( [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] with excutils.save_and_reraise_exception(): [ 711.878053] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] self.force_reraise() [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] raise self.value [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] updated_port = self._update_port( [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] _ensure_no_port_binding_failure(port) [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] raise exception.PortBindingFailed(port_id=port['id']) [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] nova.exception.PortBindingFailed: Binding failed for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384, please check neutron logs for more information. [ 711.878363] env[63593]: ERROR nova.compute.manager [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] [ 711.878656] env[63593]: DEBUG nova.compute.utils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Binding failed for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 711.879593] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.003s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 711.882635] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Build of instance aa57c935-efea-4d95-9fda-65fa9106c90e was re-scheduled: Binding failed for port c33cf0c6-f28e-40ce-8d43-7ce808cb6384, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 711.883458] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 711.883458] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Acquiring lock "refresh_cache-aa57c935-efea-4d95-9fda-65fa9106c90e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 711.883458] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Acquired lock "refresh_cache-aa57c935-efea-4d95-9fda-65fa9106c90e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 711.883610] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 712.087616] env[63593]: INFO nova.compute.manager [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] [instance: 9140518f-aae7-403d-acdd-8f8d80fc8552] Took 1.03 seconds to deallocate network for instance. [ 712.182635] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 712.408507] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 712.494014] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 712.701236] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 712.720868] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6df3fd48-806e-4a79-9b72-8fe741e2ebd9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.728257] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aaf0cf93-dd5a-4092-a969-c9f7a0aa3f59 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.757408] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2997a5c4-698e-4b49-9beb-cf60954dd4b1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.765187] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc06c488-32de-48e9-801b-627a46b85862 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.778264] env[63593]: DEBUG nova.compute.provider_tree [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 712.996817] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Releasing lock "refresh_cache-aa57c935-efea-4d95-9fda-65fa9106c90e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 712.997089] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 712.997278] env[63593]: DEBUG nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 712.997484] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 713.015960] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 713.122059] env[63593]: INFO nova.scheduler.client.report [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Deleted allocations for instance 9140518f-aae7-403d-acdd-8f8d80fc8552 [ 713.285217] env[63593]: DEBUG nova.scheduler.client.report [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 713.522368] env[63593]: DEBUG nova.network.neutron [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 713.629605] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9d8a78f1-c65f-4c75-acb5-de755aebc405 tempest-ServersAdminTestJSON-1293812331 tempest-ServersAdminTestJSON-1293812331-project-member] Lock "9140518f-aae7-403d-acdd-8f8d80fc8552" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 86.568s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.790826] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.909s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.790826] env[63593]: ERROR nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c508017e-1190-44bd-bff3-ed58a549586d, please check neutron logs for more information. [ 713.790826] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Traceback (most recent call last): [ 713.790826] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 713.790826] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self.driver.spawn(context, instance, image_meta, [ 713.790826] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 713.790826] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 713.790826] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 713.790826] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] vm_ref = self.build_virtual_machine(instance, [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] vif_infos = vmwarevif.get_vif_info(self._session, [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] for vif in network_info: [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] return self._sync_wrapper(fn, *args, **kwargs) [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self.wait() [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self[:] = self._gt.wait() [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] return self._exit_event.wait() [ 713.791144] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] current.throw(*self._exc) [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] result = function(*args, **kwargs) [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] return func(*args, **kwargs) [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] raise e [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] nwinfo = self.network_api.allocate_for_instance( [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] created_port_ids = self._update_ports_for_instance( [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 713.791524] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] with excutils.save_and_reraise_exception(): [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] self.force_reraise() [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] raise self.value [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] updated_port = self._update_port( [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] _ensure_no_port_binding_failure(port) [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] raise exception.PortBindingFailed(port_id=port['id']) [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] nova.exception.PortBindingFailed: Binding failed for port c508017e-1190-44bd-bff3-ed58a549586d, please check neutron logs for more information. [ 713.791936] env[63593]: ERROR nova.compute.manager [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] [ 713.792279] env[63593]: DEBUG nova.compute.utils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Binding failed for port c508017e-1190-44bd-bff3-ed58a549586d, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 713.797017] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Build of instance 6192c09e-64e7-4eb3-968e-922247892ca2 was re-scheduled: Binding failed for port c508017e-1190-44bd-bff3-ed58a549586d, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 713.797017] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 713.797017] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Acquiring lock "refresh_cache-6192c09e-64e7-4eb3-968e-922247892ca2" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 713.797017] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Acquired lock "refresh_cache-6192c09e-64e7-4eb3-968e-922247892ca2" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 713.797268] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 713.797268] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.946s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 714.024534] env[63593]: INFO nova.compute.manager [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] [instance: aa57c935-efea-4d95-9fda-65fa9106c90e] Took 1.03 seconds to deallocate network for instance. [ 714.131437] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 714.329473] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 714.497844] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 714.653328] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 714.751020] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f16796a5-4776-4b57-bbcd-b882c04b94a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.759126] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e6ac52b-af91-4490-af77-27d1276e0bfd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.793829] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efd46140-a2f7-4c60-8e84-c9d3a101cd2b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.801948] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b14cfe96-2070-4159-8666-473059f16704 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.818249] env[63593]: DEBUG nova.compute.provider_tree [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 715.003826] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Releasing lock "refresh_cache-6192c09e-64e7-4eb3-968e-922247892ca2" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 715.004160] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 715.004375] env[63593]: DEBUG nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 715.004581] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 715.026926] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 715.062034] env[63593]: INFO nova.scheduler.client.report [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Deleted allocations for instance aa57c935-efea-4d95-9fda-65fa9106c90e [ 715.323833] env[63593]: DEBUG nova.scheduler.client.report [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 715.530318] env[63593]: DEBUG nova.network.neutron [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 715.572744] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3dad69d4-31d2-4a36-9dae-ad1990c96cb8 tempest-ServersV294TestFqdnHostnames-186723849 tempest-ServersV294TestFqdnHostnames-186723849-project-member] Lock "aa57c935-efea-4d95-9fda-65fa9106c90e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 84.009s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 715.832335] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.037s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 715.832967] env[63593]: ERROR nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e00622e0-1125-49ce-8821-524aceb7674e, please check neutron logs for more information. [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Traceback (most recent call last): [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self.driver.spawn(context, instance, image_meta, [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self._vmops.spawn(context, instance, image_meta, injected_files, [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] vm_ref = self.build_virtual_machine(instance, [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] vif_infos = vmwarevif.get_vif_info(self._session, [ 715.832967] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] for vif in network_info: [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] return self._sync_wrapper(fn, *args, **kwargs) [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self.wait() [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self[:] = self._gt.wait() [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] return self._exit_event.wait() [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] current.throw(*self._exc) [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 715.833349] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] result = function(*args, **kwargs) [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] return func(*args, **kwargs) [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] raise e [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] nwinfo = self.network_api.allocate_for_instance( [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] created_port_ids = self._update_ports_for_instance( [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] with excutils.save_and_reraise_exception(): [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] self.force_reraise() [ 715.833664] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 715.834057] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] raise self.value [ 715.834057] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 715.834057] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] updated_port = self._update_port( [ 715.834057] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 715.834057] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] _ensure_no_port_binding_failure(port) [ 715.834057] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 715.834057] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] raise exception.PortBindingFailed(port_id=port['id']) [ 715.834057] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] nova.exception.PortBindingFailed: Binding failed for port e00622e0-1125-49ce-8821-524aceb7674e, please check neutron logs for more information. [ 715.834057] env[63593]: ERROR nova.compute.manager [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] [ 715.834057] env[63593]: DEBUG nova.compute.utils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Binding failed for port e00622e0-1125-49ce-8821-524aceb7674e, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 715.835776] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.519s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 715.837526] env[63593]: INFO nova.compute.claims [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 715.841604] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Build of instance 1205d83c-163f-41a4-ab32-a5a706763b33 was re-scheduled: Binding failed for port e00622e0-1125-49ce-8821-524aceb7674e, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 715.844910] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 715.844910] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Acquiring lock "refresh_cache-1205d83c-163f-41a4-ab32-a5a706763b33" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 715.844910] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Acquired lock "refresh_cache-1205d83c-163f-41a4-ab32-a5a706763b33" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 715.844910] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 716.035613] env[63593]: INFO nova.compute.manager [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] [instance: 6192c09e-64e7-4eb3-968e-922247892ca2] Took 1.03 seconds to deallocate network for instance. [ 716.080565] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 716.378085] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 716.519519] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 716.619090] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 717.025454] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Releasing lock "refresh_cache-1205d83c-163f-41a4-ab32-a5a706763b33" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 717.026097] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 717.028702] env[63593]: DEBUG nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 717.028702] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 717.060044] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 717.081187] env[63593]: INFO nova.scheduler.client.report [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Deleted allocations for instance 6192c09e-64e7-4eb3-968e-922247892ca2 [ 717.278205] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15442d3c-c901-4d52-ad27-37bf436af579 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.288417] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e23a556-4449-494a-8223-57ad50fc13b5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.326028] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78a83768-d904-4d98-9af1-14d3ebdceca3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.336801] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cedbe9ec-3010-41c0-b959-3542a4ccaf06 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.354099] env[63593]: DEBUG nova.compute.provider_tree [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 717.562707] env[63593]: DEBUG nova.network.neutron [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 717.592260] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ef688244-4c1e-48ff-be6e-d6e18fb704c1 tempest-ServerActionsTestOtherA-1679083544 tempest-ServerActionsTestOtherA-1679083544-project-member] Lock "6192c09e-64e7-4eb3-968e-922247892ca2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 77.450s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 717.858619] env[63593]: DEBUG nova.scheduler.client.report [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 718.068158] env[63593]: INFO nova.compute.manager [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] [instance: 1205d83c-163f-41a4-ab32-a5a706763b33] Took 1.04 seconds to deallocate network for instance. [ 718.095554] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 718.368042] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.533s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.368696] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 718.374846] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.403s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 718.630152] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 718.883264] env[63593]: DEBUG nova.compute.utils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 718.887861] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 718.888501] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 718.940306] env[63593]: DEBUG nova.policy [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6880a0273fa34ddb8d4f32069f8b0977', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'aa7ef13c10004d26a37ddb89cfebda6f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 719.118708] env[63593]: INFO nova.scheduler.client.report [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Deleted allocations for instance 1205d83c-163f-41a4-ab32-a5a706763b33 [ 719.317382] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Acquiring lock "94486acf-7f7b-4187-b9d1-332f70488bd5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 719.317686] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Lock "94486acf-7f7b-4187-b9d1-332f70488bd5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 719.361370] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76adbd93-59cb-4982-9e43-27d920c7b342 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.370119] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffef4186-a005-43db-8823-22f02fe6ecbb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.408676] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 719.414122] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff67d179-959f-40c4-872e-1e5fdc712d40 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.422437] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5092b40a-4ec5-4405-8b3e-491258db1772 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.441586] env[63593]: DEBUG nova.compute.provider_tree [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 719.557783] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Successfully created port: f3bc3473-0998-475d-aa09-c66d9fc6eb0e {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 719.628370] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6d8a6129-f1b0-4114-9501-edb722930a71 tempest-ServerActionsTestOtherB-1703475404 tempest-ServerActionsTestOtherB-1703475404-project-member] Lock "1205d83c-163f-41a4-ab32-a5a706763b33" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 79.162s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 719.944123] env[63593]: DEBUG nova.scheduler.client.report [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 720.132061] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 720.426672] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 720.451607] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.076s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.452011] env[63593]: ERROR nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 720.452011] env[63593]: Faults: ['InvalidArgument'] [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Traceback (most recent call last): [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] self.driver.spawn(context, instance, image_meta, [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] self._fetch_image_if_missing(context, vi) [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] image_cache(vi, tmp_image_ds_loc) [ 720.452011] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] vm_util.copy_virtual_disk( [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] session._wait_for_task(vmdk_copy_task) [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] return self.wait_for_task(task_ref) [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] return evt.wait() [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] result = hub.switch() [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] return self.greenlet.switch() [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 720.452410] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] self.f(*self.args, **self.kw) [ 720.452820] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 720.452820] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] raise exceptions.translate_fault(task_info.error) [ 720.452820] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 720.452820] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Faults: ['InvalidArgument'] [ 720.452820] env[63593]: ERROR nova.compute.manager [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] [ 720.452820] env[63593]: DEBUG nova.compute.utils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 720.454305] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.426s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 720.465017] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Build of instance 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e was re-scheduled: A specified parameter was not correct: fileType [ 720.465017] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 720.465567] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 720.465707] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquiring lock "refresh_cache-65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 720.465837] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Acquired lock "refresh_cache-65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 720.466135] env[63593]: DEBUG nova.network.neutron [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 720.481234] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 720.481515] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 720.481589] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 720.481738] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 720.481902] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 720.484380] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 720.485527] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 720.485789] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 720.485882] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 720.486060] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 720.488423] env[63593]: DEBUG nova.virt.hardware [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 720.488628] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc04838e-8023-47a8-9618-fcbda9933db3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 720.500327] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-996cdf73-998f-45a4-9000-a0130833681e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 720.663161] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.014708] env[63593]: DEBUG nova.compute.manager [req-d123e0f4-bf08-4b62-9e24-d7f629c87028 req-cdd9cc57-fe61-4e7a-b1b9-ba7adcea7e87 service nova] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Received event network-changed-f3bc3473-0998-475d-aa09-c66d9fc6eb0e {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 721.014948] env[63593]: DEBUG nova.compute.manager [req-d123e0f4-bf08-4b62-9e24-d7f629c87028 req-cdd9cc57-fe61-4e7a-b1b9-ba7adcea7e87 service nova] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Refreshing instance network info cache due to event network-changed-f3bc3473-0998-475d-aa09-c66d9fc6eb0e. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 721.015140] env[63593]: DEBUG oslo_concurrency.lockutils [req-d123e0f4-bf08-4b62-9e24-d7f629c87028 req-cdd9cc57-fe61-4e7a-b1b9-ba7adcea7e87 service nova] Acquiring lock "refresh_cache-3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 721.015636] env[63593]: DEBUG oslo_concurrency.lockutils [req-d123e0f4-bf08-4b62-9e24-d7f629c87028 req-cdd9cc57-fe61-4e7a-b1b9-ba7adcea7e87 service nova] Acquired lock "refresh_cache-3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 721.015925] env[63593]: DEBUG nova.network.neutron [req-d123e0f4-bf08-4b62-9e24-d7f629c87028 req-cdd9cc57-fe61-4e7a-b1b9-ba7adcea7e87 service nova] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Refreshing network info cache for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 721.019485] env[63593]: DEBUG nova.network.neutron [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 721.138769] env[63593]: ERROR nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e, please check neutron logs for more information. [ 721.138769] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 721.138769] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 721.138769] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 721.138769] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 721.138769] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 721.138769] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 721.138769] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 721.138769] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 721.138769] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 721.138769] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 721.138769] env[63593]: ERROR nova.compute.manager raise self.value [ 721.138769] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 721.138769] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 721.138769] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 721.138769] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 721.139522] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 721.139522] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 721.139522] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e, please check neutron logs for more information. [ 721.139522] env[63593]: ERROR nova.compute.manager [ 721.139522] env[63593]: Traceback (most recent call last): [ 721.139522] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 721.139522] env[63593]: listener.cb(fileno) [ 721.139522] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 721.139522] env[63593]: result = function(*args, **kwargs) [ 721.139522] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 721.139522] env[63593]: return func(*args, **kwargs) [ 721.139522] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 721.139522] env[63593]: raise e [ 721.139522] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 721.139522] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 721.139522] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 721.139522] env[63593]: created_port_ids = self._update_ports_for_instance( [ 721.139522] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 721.139522] env[63593]: with excutils.save_and_reraise_exception(): [ 721.139522] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 721.139522] env[63593]: self.force_reraise() [ 721.139522] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 721.139522] env[63593]: raise self.value [ 721.139522] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 721.139522] env[63593]: updated_port = self._update_port( [ 721.139522] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 721.139522] env[63593]: _ensure_no_port_binding_failure(port) [ 721.139522] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 721.139522] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 721.141115] env[63593]: nova.exception.PortBindingFailed: Binding failed for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e, please check neutron logs for more information. [ 721.141115] env[63593]: Removing descriptor: 19 [ 721.141115] env[63593]: ERROR nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e, please check neutron logs for more information. [ 721.141115] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Traceback (most recent call last): [ 721.141115] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 721.141115] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] yield resources [ 721.141115] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 721.141115] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self.driver.spawn(context, instance, image_meta, [ 721.141115] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 721.141115] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self._vmops.spawn(context, instance, image_meta, injected_files, [ 721.141115] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 721.141115] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] vm_ref = self.build_virtual_machine(instance, [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] vif_infos = vmwarevif.get_vif_info(self._session, [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] for vif in network_info: [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] return self._sync_wrapper(fn, *args, **kwargs) [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self.wait() [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self[:] = self._gt.wait() [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] return self._exit_event.wait() [ 721.141588] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] result = hub.switch() [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] return self.greenlet.switch() [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] result = function(*args, **kwargs) [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] return func(*args, **kwargs) [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] raise e [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] nwinfo = self.network_api.allocate_for_instance( [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 721.142067] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] created_port_ids = self._update_ports_for_instance( [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] with excutils.save_and_reraise_exception(): [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self.force_reraise() [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] raise self.value [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] updated_port = self._update_port( [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] _ensure_no_port_binding_failure(port) [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 721.142493] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] raise exception.PortBindingFailed(port_id=port['id']) [ 721.142966] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] nova.exception.PortBindingFailed: Binding failed for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e, please check neutron logs for more information. [ 721.142966] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] [ 721.142966] env[63593]: INFO nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Terminating instance [ 721.145759] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "refresh_cache-3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 721.204155] env[63593]: DEBUG nova.network.neutron [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 721.434432] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "94ff0182-d1ce-405f-bdb5-a2399bd0f4de" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.435090] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "94ff0182-d1ce-405f-bdb5-a2399bd0f4de" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 721.491032] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1135b59a-ef13-4657-98a1-08e8e26661c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.498692] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-face456c-2673-4172-9e47-affc93bdb178 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.534102] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a2241a4-46d8-4a6b-9e2b-9ed530a4e95f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.541399] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7a7ce97-85d2-47eb-b160-37730cf2d4a8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.556450] env[63593]: DEBUG nova.compute.provider_tree [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 721.558466] env[63593]: DEBUG nova.network.neutron [req-d123e0f4-bf08-4b62-9e24-d7f629c87028 req-cdd9cc57-fe61-4e7a-b1b9-ba7adcea7e87 service nova] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 721.707623] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Releasing lock "refresh_cache-65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 721.707623] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 721.707623] env[63593]: DEBUG nova.compute.manager [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] [instance: 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 721.720360] env[63593]: DEBUG nova.network.neutron [req-d123e0f4-bf08-4b62-9e24-d7f629c87028 req-cdd9cc57-fe61-4e7a-b1b9-ba7adcea7e87 service nova] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 721.799367] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "4e312fd5-01bf-4bd4-8a61-26624ae05036" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.799768] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "4e312fd5-01bf-4bd4-8a61-26624ae05036" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 722.064157] env[63593]: DEBUG nova.scheduler.client.report [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 722.223534] env[63593]: DEBUG oslo_concurrency.lockutils [req-d123e0f4-bf08-4b62-9e24-d7f629c87028 req-cdd9cc57-fe61-4e7a-b1b9-ba7adcea7e87 service nova] Releasing lock "refresh_cache-3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 722.223920] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquired lock "refresh_cache-3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 722.224115] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 722.571949] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.118s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 722.572481] env[63593]: ERROR nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc, please check neutron logs for more information. [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Traceback (most recent call last): [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self.driver.spawn(context, instance, image_meta, [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] vm_ref = self.build_virtual_machine(instance, [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] vif_infos = vmwarevif.get_vif_info(self._session, [ 722.572481] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] for vif in network_info: [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] return self._sync_wrapper(fn, *args, **kwargs) [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self.wait() [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self[:] = self._gt.wait() [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] return self._exit_event.wait() [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] current.throw(*self._exc) [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 722.572793] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] result = function(*args, **kwargs) [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] return func(*args, **kwargs) [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] raise e [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] nwinfo = self.network_api.allocate_for_instance( [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] created_port_ids = self._update_ports_for_instance( [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] with excutils.save_and_reraise_exception(): [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] self.force_reraise() [ 722.573150] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 722.573469] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] raise self.value [ 722.573469] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 722.573469] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] updated_port = self._update_port( [ 722.573469] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 722.573469] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] _ensure_no_port_binding_failure(port) [ 722.573469] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 722.573469] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] raise exception.PortBindingFailed(port_id=port['id']) [ 722.573469] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] nova.exception.PortBindingFailed: Binding failed for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc, please check neutron logs for more information. [ 722.573469] env[63593]: ERROR nova.compute.manager [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] [ 722.573469] env[63593]: DEBUG nova.compute.utils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Binding failed for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 722.574882] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.840s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 722.579163] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Build of instance 6bf23f6a-6727-448f-80d9-fe954addd49e was re-scheduled: Binding failed for port 6a9869ab-f8a7-4643-9a58-8a69651e1bbc, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 722.581912] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 722.582171] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Acquiring lock "refresh_cache-6bf23f6a-6727-448f-80d9-fe954addd49e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 722.582322] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Acquired lock "refresh_cache-6bf23f6a-6727-448f-80d9-fe954addd49e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 722.582480] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 722.742599] env[63593]: INFO nova.scheduler.client.report [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Deleted allocations for instance 65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e [ 722.749278] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 722.832175] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 723.109110] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 723.133556] env[63593]: DEBUG nova.compute.manager [req-cc4ab239-d1c7-4a2f-9362-371fbcdb87cb req-89a23cdd-c410-4593-914e-3bc0edf2445a service nova] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Received event network-vif-deleted-f3bc3473-0998-475d-aa09-c66d9fc6eb0e {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 723.226559] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 723.255198] env[63593]: DEBUG oslo_concurrency.lockutils [None req-93ac8f9a-3390-451c-8a93-f138f56bbe7b tempest-ServerDiagnosticsV248Test-1702219166 tempest-ServerDiagnosticsV248Test-1702219166-project-member] Lock "65e7ceb8-2b6a-4d36-b698-6e15a4f17f2e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 116.131s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.339480] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Releasing lock "refresh_cache-3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 723.340150] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 723.340653] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 723.341446] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5c95fa68-418d-4a17-9308-8b8ed4f9cbcf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 723.351311] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2dca0538-6697-4e22-9a1a-460bfc387e7b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 723.376838] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff could not be found. [ 723.376838] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 723.376909] env[63593]: INFO nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Took 0.04 seconds to destroy the instance on the hypervisor. [ 723.377245] env[63593]: DEBUG oslo.service.loopingcall [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 723.380122] env[63593]: DEBUG nova.compute.manager [-] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 723.380251] env[63593]: DEBUG nova.network.neutron [-] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 723.399543] env[63593]: DEBUG nova.network.neutron [-] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 723.519142] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfd3651d-1685-430b-91da-c67ab7aaadf0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 723.526027] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d10c27bc-584d-4d78-a6ae-2df7313eaa60 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 723.559336] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01c30f7a-7b0e-45bd-b7cf-94957aade263 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 723.568207] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c63db0d-e3a3-419d-946c-376c62e106da {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 723.584625] env[63593]: DEBUG nova.compute.provider_tree [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 723.731349] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Releasing lock "refresh_cache-6bf23f6a-6727-448f-80d9-fe954addd49e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 723.731648] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 723.731842] env[63593]: DEBUG nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 723.732022] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 723.758317] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 723.764935] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 723.902100] env[63593]: DEBUG nova.network.neutron [-] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 724.087943] env[63593]: DEBUG nova.scheduler.client.report [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 724.269258] env[63593]: DEBUG nova.network.neutron [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 724.297422] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 724.404841] env[63593]: INFO nova.compute.manager [-] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Took 1.02 seconds to deallocate network for instance. [ 724.408358] env[63593]: DEBUG nova.compute.claims [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 724.408611] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 724.595525] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.021s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 724.596312] env[63593]: ERROR nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a12681e7-678c-4636-a5b1-9d47074e6198, please check neutron logs for more information. [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Traceback (most recent call last): [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self.driver.spawn(context, instance, image_meta, [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self._vmops.spawn(context, instance, image_meta, injected_files, [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] vm_ref = self.build_virtual_machine(instance, [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] vif_infos = vmwarevif.get_vif_info(self._session, [ 724.596312] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] for vif in network_info: [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] return self._sync_wrapper(fn, *args, **kwargs) [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self.wait() [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self[:] = self._gt.wait() [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] return self._exit_event.wait() [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] current.throw(*self._exc) [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 724.597202] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] result = function(*args, **kwargs) [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] return func(*args, **kwargs) [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] raise e [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] nwinfo = self.network_api.allocate_for_instance( [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] created_port_ids = self._update_ports_for_instance( [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] with excutils.save_and_reraise_exception(): [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] self.force_reraise() [ 724.598127] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 724.599057] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] raise self.value [ 724.599057] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 724.599057] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] updated_port = self._update_port( [ 724.599057] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 724.599057] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] _ensure_no_port_binding_failure(port) [ 724.599057] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 724.599057] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] raise exception.PortBindingFailed(port_id=port['id']) [ 724.599057] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] nova.exception.PortBindingFailed: Binding failed for port a12681e7-678c-4636-a5b1-9d47074e6198, please check neutron logs for more information. [ 724.599057] env[63593]: ERROR nova.compute.manager [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] [ 724.599057] env[63593]: DEBUG nova.compute.utils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Binding failed for port a12681e7-678c-4636-a5b1-9d47074e6198, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 724.599844] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 14.584s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 724.600294] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Build of instance 6462167e-ae83-4d73-bfb2-ec7024e56a07 was re-scheduled: Binding failed for port a12681e7-678c-4636-a5b1-9d47074e6198, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 724.600732] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 724.601017] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "refresh_cache-6462167e-ae83-4d73-bfb2-ec7024e56a07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 724.601199] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "refresh_cache-6462167e-ae83-4d73-bfb2-ec7024e56a07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 724.601865] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 724.782237] env[63593]: INFO nova.compute.manager [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] [instance: 6bf23f6a-6727-448f-80d9-fe954addd49e] Took 1.05 seconds to deallocate network for instance. [ 725.122691] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 725.233871] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 725.382915] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Acquiring lock "43b2aa92-913e-4bae-9e25-82cf423d4148" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 725.383204] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Lock "43b2aa92-913e-4bae-9e25-82cf423d4148" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 725.740230] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "refresh_cache-6462167e-ae83-4d73-bfb2-ec7024e56a07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 725.740230] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 725.740230] env[63593]: DEBUG nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 725.740230] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 725.783091] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 725.816098] env[63593]: INFO nova.scheduler.client.report [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Deleted allocations for instance 6bf23f6a-6727-448f-80d9-fe954addd49e [ 726.148727] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6bf23f6a-6727-448f-80d9-fe954addd49e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 726.286682] env[63593]: DEBUG nova.network.neutron [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 726.323985] env[63593]: DEBUG oslo_concurrency.lockutils [None req-14b022ad-6e5a-45f1-beb6-b7a1cebb3285 tempest-ServersTestFqdnHostnames-1641381201 tempest-ServersTestFqdnHostnames-1641381201-project-member] Lock "6bf23f6a-6727-448f-80d9-fe954addd49e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 83.907s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 726.655396] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6462167e-ae83-4d73-bfb2-ec7024e56a07 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 726.655758] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 04de495d-8896-4c6d-a059-9a80477e3b07 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 726.655758] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 726.791769] env[63593]: INFO nova.compute.manager [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 6462167e-ae83-4d73-bfb2-ec7024e56a07] Took 1.05 seconds to deallocate network for instance. [ 726.830346] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 727.159527] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 83204968-ba8d-4e40-9f3b-d6d427fc5bb5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 727.357799] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 727.663902] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 40449f7d-3a21-487a-a778-43ccd7b23949 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 727.843422] env[63593]: INFO nova.scheduler.client.report [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleted allocations for instance 6462167e-ae83-4d73-bfb2-ec7024e56a07 [ 728.170374] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 70595430-6ab6-49d4-b8fe-131bd44de838 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 728.354468] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4bc17dd3-a005-481c-8365-a0eae1d62dcc tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "6462167e-ae83-4d73-bfb2-ec7024e56a07" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 85.920s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 728.673266] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 006d2ab8-e38b-4b69-b3b2-115c39e44218 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 728.709968] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Acquiring lock "be01dd8d-53fb-4fcc-9bf8-41934b4d0263" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 728.709968] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Lock "be01dd8d-53fb-4fcc-9bf8-41934b4d0263" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 728.859520] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 729.179439] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance eddf892e-9706-4d33-8241-f8df6c29dee4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 729.385642] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.682639] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance c99cbfae-db59-4b84-b8ab-09b84ebe4623 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 730.187410] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4f1c0deb-5759-44d2-a2ef-f94e1905949d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 730.690466] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4da891e9-39e0-4739-b1b9-fb438c1111ab has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 730.853382] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "2b478998-632e-4286-8d97-eab4f025bdda" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 730.853618] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "2b478998-632e-4286-8d97-eab4f025bdda" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 731.195753] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 788e5910-c189-4d13-a2fa-e5d6e915e95d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 731.698723] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b9bd0d9b-322f-476a-ae3f-792ccd869529 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 731.745319] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Acquiring lock "464b651f-3f3f-425d-9da6-546ea9610515" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 731.745549] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "464b651f-3f3f-425d-9da6-546ea9610515" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 732.201974] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 9bc65ce3-6459-4356-8a63-74f55b6f3b47 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 732.704686] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b62cc771-973a-4d23-87b4-825211494d9c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 733.208261] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b965361a-6b76-4836-9887-b59ca990fd67 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 733.711232] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 077acb40-d01b-42ad-aa68-cbffc4087100 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 734.213849] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 1db3e15b-3168-4d01-930f-fe8008b95df6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 734.717849] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b27b2ed8-ec99-4962-8742-3a8264906b47 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 735.221088] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a269ba6a-8439-448f-ad51-e3084c89c2a5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 735.724338] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 937998a3-dac3-4ce5-b363-7ceee313531a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 736.228415] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b5889795-9995-4211-8b3f-8d14680fde6f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 736.731885] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 737.234657] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 94486acf-7f7b-4187-b9d1-332f70488bd5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 737.737775] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 94ff0182-d1ce-405f-bdb5-a2399bd0f4de has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 738.241379] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e312fd5-01bf-4bd4-8a61-26624ae05036 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 738.744420] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 43b2aa92-913e-4bae-9e25-82cf423d4148 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 738.744642] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 2 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 738.744642] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=896MB phys_disk=200GB used_disk=2GB total_vcpus=48 used_vcpus=2 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 739.055778] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-847e2a54-0d97-4f14-8d0b-4100c749a057 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 739.063468] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccd963aa-bca8-4ea7-9793-10c2bb400cba {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 739.093345] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a238c0b7-e5be-425e-b78d-5598a9b4a11d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 739.100404] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e994ee0-f581-4527-a0ce-f2a33e3b3272 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 739.113721] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 739.616763] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 740.121421] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 740.121710] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 15.523s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 740.121947] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 29.873s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 740.915176] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76304d0e-dedf-458f-ac8a-1ca935bceb77 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.922663] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aacbd0f0-3bcc-4f55-98fb-97554cbf6910 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.951453] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd60a51b-a1fa-45ad-a13e-fb6390c18687 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.958412] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7830cdfb-2892-4818-967a-c7df570101da {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.970862] env[63593]: DEBUG nova.compute.provider_tree [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 741.474646] env[63593]: DEBUG nova.scheduler.client.report [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 741.980666] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.859s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 741.981373] env[63593]: ERROR nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1b2017a6-6125-4b40-9e28-1d038ee03a57, please check neutron logs for more information. [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Traceback (most recent call last): [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self.driver.spawn(context, instance, image_meta, [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self._vmops.spawn(context, instance, image_meta, injected_files, [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] vm_ref = self.build_virtual_machine(instance, [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] vif_infos = vmwarevif.get_vif_info(self._session, [ 741.981373] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] for vif in network_info: [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] return self._sync_wrapper(fn, *args, **kwargs) [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self.wait() [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self[:] = self._gt.wait() [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] return self._exit_event.wait() [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] current.throw(*self._exc) [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 741.981646] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] result = function(*args, **kwargs) [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] return func(*args, **kwargs) [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] raise e [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] nwinfo = self.network_api.allocate_for_instance( [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] created_port_ids = self._update_ports_for_instance( [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] with excutils.save_and_reraise_exception(): [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] self.force_reraise() [ 741.981947] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 741.982374] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] raise self.value [ 741.982374] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 741.982374] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] updated_port = self._update_port( [ 741.982374] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 741.982374] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] _ensure_no_port_binding_failure(port) [ 741.982374] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 741.982374] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] raise exception.PortBindingFailed(port_id=port['id']) [ 741.982374] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] nova.exception.PortBindingFailed: Binding failed for port 1b2017a6-6125-4b40-9e28-1d038ee03a57, please check neutron logs for more information. [ 741.982374] env[63593]: ERROR nova.compute.manager [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] [ 741.982374] env[63593]: DEBUG nova.compute.utils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Binding failed for port 1b2017a6-6125-4b40-9e28-1d038ee03a57, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 741.983394] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.282s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 741.984952] env[63593]: INFO nova.compute.claims [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 741.988347] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Build of instance 04de495d-8896-4c6d-a059-9a80477e3b07 was re-scheduled: Binding failed for port 1b2017a6-6125-4b40-9e28-1d038ee03a57, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 741.988451] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 741.988662] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "refresh_cache-04de495d-8896-4c6d-a059-9a80477e3b07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 741.988808] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquired lock "refresh_cache-04de495d-8896-4c6d-a059-9a80477e3b07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 741.988962] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 742.508711] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 742.596820] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 743.100798] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Releasing lock "refresh_cache-04de495d-8896-4c6d-a059-9a80477e3b07" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 743.100798] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 743.100941] env[63593]: DEBUG nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 743.101100] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 743.119769] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 743.318317] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70a10a62-adff-45d3-897e-0dcdc7ca11f3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.325807] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4077316-0350-4b1f-ba7b-5488a02c6868 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.355249] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a596c36a-aa35-43ee-9c9d-67695efcc7f0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.362480] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8883729-1981-4e96-a034-8bb61a5d89ce {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.375395] env[63593]: DEBUG nova.compute.provider_tree [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 743.623147] env[63593]: DEBUG nova.network.neutron [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 743.878694] env[63593]: DEBUG nova.scheduler.client.report [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 744.126141] env[63593]: INFO nova.compute.manager [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 04de495d-8896-4c6d-a059-9a80477e3b07] Took 1.02 seconds to deallocate network for instance. [ 744.383624] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.400s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.384073] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 744.386865] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.734s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.388240] env[63593]: INFO nova.compute.claims [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 744.893167] env[63593]: DEBUG nova.compute.utils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 744.897324] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 744.897324] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 744.946689] env[63593]: DEBUG nova.policy [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ecdcb1f0231c421c891a42f9b5d4c6ae', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ed8027cebe3441248b0a20ec14459879', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 745.152509] env[63593]: INFO nova.scheduler.client.report [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Deleted allocations for instance 04de495d-8896-4c6d-a059-9a80477e3b07 [ 745.233575] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Successfully created port: 0108f921-dc23-4118-ab51-5b67dd87d26f {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 745.397732] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 745.660879] env[63593]: DEBUG oslo_concurrency.lockutils [None req-79493184-7663-455e-b93f-1449cc3cd121 tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "04de495d-8896-4c6d-a059-9a80477e3b07" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 101.471s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 745.791591] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85124119-2e64-4860-b7ec-bcaaf06917b8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.800362] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a064a1e0-ef33-4ea3-ac12-45fb610912e6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.842360] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76fc6041-dad2-4eee-bc19-1713c81268ad {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.849654] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-082dce42-a0f8-43e0-be69-349ab8912a16 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.867107] env[63593]: DEBUG nova.compute.provider_tree [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 745.906176] env[63593]: INFO nova.virt.block_device [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Booting with volume d9498ead-0475-49b3-87f2-16cdfb41c5af at /dev/sda [ 745.956985] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2d5fcbf5-5870-4c47-a567-a6f5bc8a8678 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.965016] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b60675e-436f-48d4-9694-5185f0f424b7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.986471] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-65d88cfe-e8f2-4633-adbf-8e76b5667d27 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.996771] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-245e6e7d-b13d-4a33-8d14-f3a40ed092e5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.019712] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-611fd005-32c1-421a-8608-6f7e61666132 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.023984] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a251f4ec-72a2-4643-aab5-43cbd40e6620 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.040965] env[63593]: DEBUG nova.virt.block_device [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Updating existing volume attachment record: f81492ad-2cc1-414c-8291-557ff8adbed7 {{(pid=63593) _volume_attach /opt/stack/nova/nova/virt/block_device.py:665}} [ 746.161280] env[63593]: DEBUG nova.compute.manager [req-0090a002-b9b0-4288-b6d6-a56dc2f15656 req-6446a475-3830-4a09-bd13-df49e3ae999c service nova] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Received event network-changed-0108f921-dc23-4118-ab51-5b67dd87d26f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 746.161280] env[63593]: DEBUG nova.compute.manager [req-0090a002-b9b0-4288-b6d6-a56dc2f15656 req-6446a475-3830-4a09-bd13-df49e3ae999c service nova] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Refreshing instance network info cache due to event network-changed-0108f921-dc23-4118-ab51-5b67dd87d26f. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 746.161280] env[63593]: DEBUG oslo_concurrency.lockutils [req-0090a002-b9b0-4288-b6d6-a56dc2f15656 req-6446a475-3830-4a09-bd13-df49e3ae999c service nova] Acquiring lock "refresh_cache-83204968-ba8d-4e40-9f3b-d6d427fc5bb5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 746.161280] env[63593]: DEBUG oslo_concurrency.lockutils [req-0090a002-b9b0-4288-b6d6-a56dc2f15656 req-6446a475-3830-4a09-bd13-df49e3ae999c service nova] Acquired lock "refresh_cache-83204968-ba8d-4e40-9f3b-d6d427fc5bb5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 746.161280] env[63593]: DEBUG nova.network.neutron [req-0090a002-b9b0-4288-b6d6-a56dc2f15656 req-6446a475-3830-4a09-bd13-df49e3ae999c service nova] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Refreshing network info cache for port 0108f921-dc23-4118-ab51-5b67dd87d26f {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 746.172023] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 746.369978] env[63593]: DEBUG nova.scheduler.client.report [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 746.465988] env[63593]: ERROR nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0108f921-dc23-4118-ab51-5b67dd87d26f, please check neutron logs for more information. [ 746.465988] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 746.465988] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 746.465988] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 746.465988] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 746.465988] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 746.465988] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 746.465988] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 746.465988] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 746.465988] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 746.465988] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 746.465988] env[63593]: ERROR nova.compute.manager raise self.value [ 746.465988] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 746.465988] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 746.465988] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 746.465988] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 746.466433] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 746.466433] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 746.466433] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0108f921-dc23-4118-ab51-5b67dd87d26f, please check neutron logs for more information. [ 746.466433] env[63593]: ERROR nova.compute.manager [ 746.466433] env[63593]: Traceback (most recent call last): [ 746.466433] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 746.466433] env[63593]: listener.cb(fileno) [ 746.466433] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 746.466433] env[63593]: result = function(*args, **kwargs) [ 746.466433] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 746.466433] env[63593]: return func(*args, **kwargs) [ 746.466433] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 746.466433] env[63593]: raise e [ 746.466433] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 746.466433] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 746.466433] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 746.466433] env[63593]: created_port_ids = self._update_ports_for_instance( [ 746.466433] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 746.466433] env[63593]: with excutils.save_and_reraise_exception(): [ 746.466433] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 746.466433] env[63593]: self.force_reraise() [ 746.466433] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 746.466433] env[63593]: raise self.value [ 746.466433] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 746.466433] env[63593]: updated_port = self._update_port( [ 746.466433] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 746.466433] env[63593]: _ensure_no_port_binding_failure(port) [ 746.466433] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 746.466433] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 746.467125] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 0108f921-dc23-4118-ab51-5b67dd87d26f, please check neutron logs for more information. [ 746.467125] env[63593]: Removing descriptor: 21 [ 746.685273] env[63593]: DEBUG nova.network.neutron [req-0090a002-b9b0-4288-b6d6-a56dc2f15656 req-6446a475-3830-4a09-bd13-df49e3ae999c service nova] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 746.693764] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.745184] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "9385f942-728d-4041-bc52-77e3c9752b7f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.745403] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "9385f942-728d-4041-bc52-77e3c9752b7f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.769600] env[63593]: DEBUG nova.network.neutron [req-0090a002-b9b0-4288-b6d6-a56dc2f15656 req-6446a475-3830-4a09-bd13-df49e3ae999c service nova] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 746.875075] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.488s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.875644] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 746.878340] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 30.262s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.879868] env[63593]: INFO nova.compute.claims [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 747.272255] env[63593]: DEBUG oslo_concurrency.lockutils [req-0090a002-b9b0-4288-b6d6-a56dc2f15656 req-6446a475-3830-4a09-bd13-df49e3ae999c service nova] Releasing lock "refresh_cache-83204968-ba8d-4e40-9f3b-d6d427fc5bb5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 747.385398] env[63593]: DEBUG nova.compute.utils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 747.388689] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 747.388913] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 747.442429] env[63593]: DEBUG nova.policy [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7bc29b5ff2c64d32b13d79094cc44edd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'a61073725653448197793ad2ddaf86e2', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 747.732119] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Successfully created port: 8482d961-e7b9-4b67-9b93-5a63fdc65d68 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 747.889803] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 748.157019] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 748.157019] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 748.157019] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 748.157019] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 748.157279] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 748.157279] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 748.157279] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 748.157279] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 748.157279] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 748.157411] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 748.157411] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 748.157411] env[63593]: DEBUG nova.virt.hardware [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 748.157411] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51639546-f39d-41df-897e-20c09eaf6d71 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.170414] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-434dbd19-aaee-41cd-b3ff-47fb338291d2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.191591] env[63593]: ERROR nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0108f921-dc23-4118-ab51-5b67dd87d26f, please check neutron logs for more information. [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Traceback (most recent call last): [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] yield resources [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self.driver.spawn(context, instance, image_meta, [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] vm_ref = self.build_virtual_machine(instance, [ 748.191591] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] vif_infos = vmwarevif.get_vif_info(self._session, [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] for vif in network_info: [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] return self._sync_wrapper(fn, *args, **kwargs) [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self.wait() [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self[:] = self._gt.wait() [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] return self._exit_event.wait() [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 748.191906] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] current.throw(*self._exc) [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] result = function(*args, **kwargs) [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] return func(*args, **kwargs) [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] raise e [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] nwinfo = self.network_api.allocate_for_instance( [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] created_port_ids = self._update_ports_for_instance( [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] with excutils.save_and_reraise_exception(): [ 748.192244] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self.force_reraise() [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] raise self.value [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] updated_port = self._update_port( [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] _ensure_no_port_binding_failure(port) [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] raise exception.PortBindingFailed(port_id=port['id']) [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] nova.exception.PortBindingFailed: Binding failed for port 0108f921-dc23-4118-ab51-5b67dd87d26f, please check neutron logs for more information. [ 748.192567] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] [ 748.192567] env[63593]: INFO nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Terminating instance [ 748.198525] env[63593]: DEBUG nova.compute.manager [req-cc18390c-267e-498a-8933-1d5d672c5a0d req-9f23bc28-6ae5-4904-827e-6869460e0600 service nova] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Received event network-vif-deleted-0108f921-dc23-4118-ab51-5b67dd87d26f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 748.199117] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Acquiring lock "refresh_cache-83204968-ba8d-4e40-9f3b-d6d427fc5bb5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 748.199324] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Acquired lock "refresh_cache-83204968-ba8d-4e40-9f3b-d6d427fc5bb5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 748.199529] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 748.288223] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-290a5d61-20ea-40d1-956b-cbb44ed63563 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.293746] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c513ced-c42d-41b7-888a-0ddeac96ce77 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.324353] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f94225b-cc26-465e-9772-2e5476ad6a0b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.332169] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a22ae21-66ee-45de-a5ba-8c4a4f116e68 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.353762] env[63593]: DEBUG nova.compute.provider_tree [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 748.697104] env[63593]: ERROR nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68, please check neutron logs for more information. [ 748.697104] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 748.697104] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.697104] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 748.697104] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.697104] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 748.697104] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.697104] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 748.697104] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.697104] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 748.697104] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.697104] env[63593]: ERROR nova.compute.manager raise self.value [ 748.697104] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.697104] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 748.697104] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.697104] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 748.697598] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.697598] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 748.697598] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68, please check neutron logs for more information. [ 748.697598] env[63593]: ERROR nova.compute.manager [ 748.697598] env[63593]: Traceback (most recent call last): [ 748.697598] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 748.697598] env[63593]: listener.cb(fileno) [ 748.697598] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.697598] env[63593]: result = function(*args, **kwargs) [ 748.697598] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 748.697598] env[63593]: return func(*args, **kwargs) [ 748.697598] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.697598] env[63593]: raise e [ 748.697598] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.697598] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 748.697598] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.697598] env[63593]: created_port_ids = self._update_ports_for_instance( [ 748.697598] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.697598] env[63593]: with excutils.save_and_reraise_exception(): [ 748.697598] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.697598] env[63593]: self.force_reraise() [ 748.697598] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.697598] env[63593]: raise self.value [ 748.697598] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.697598] env[63593]: updated_port = self._update_port( [ 748.697598] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.697598] env[63593]: _ensure_no_port_binding_failure(port) [ 748.697598] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.697598] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 748.698527] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68, please check neutron logs for more information. [ 748.698527] env[63593]: Removing descriptor: 21 [ 748.722998] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 748.794226] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 748.856314] env[63593]: DEBUG nova.scheduler.client.report [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 748.901673] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 748.925964] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 748.926222] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 748.926376] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 748.926552] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 748.926694] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 748.926897] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 748.927151] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 748.927314] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 748.927482] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 748.927642] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 748.927860] env[63593]: DEBUG nova.virt.hardware [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 748.928708] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcc4aa61-b9ec-46bd-8e8b-b9036a26ed9a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.936348] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64628b2d-188d-4087-8683-2519c9b3be63 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.951071] env[63593]: ERROR nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68, please check neutron logs for more information. [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Traceback (most recent call last): [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] yield resources [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self.driver.spawn(context, instance, image_meta, [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self._vmops.spawn(context, instance, image_meta, injected_files, [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] vm_ref = self.build_virtual_machine(instance, [ 748.951071] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] vif_infos = vmwarevif.get_vif_info(self._session, [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] for vif in network_info: [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] return self._sync_wrapper(fn, *args, **kwargs) [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self.wait() [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self[:] = self._gt.wait() [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] return self._exit_event.wait() [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 748.951373] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] current.throw(*self._exc) [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] result = function(*args, **kwargs) [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] return func(*args, **kwargs) [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] raise e [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] nwinfo = self.network_api.allocate_for_instance( [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] created_port_ids = self._update_ports_for_instance( [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] with excutils.save_and_reraise_exception(): [ 748.951681] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self.force_reraise() [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] raise self.value [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] updated_port = self._update_port( [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] _ensure_no_port_binding_failure(port) [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] raise exception.PortBindingFailed(port_id=port['id']) [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] nova.exception.PortBindingFailed: Binding failed for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68, please check neutron logs for more information. [ 748.952062] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] [ 748.952062] env[63593]: INFO nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Terminating instance [ 748.953812] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Acquiring lock "refresh_cache-40449f7d-3a21-487a-a778-43ccd7b23949" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 748.953812] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Acquired lock "refresh_cache-40449f7d-3a21-487a-a778-43ccd7b23949" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 748.953812] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 749.299648] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Releasing lock "refresh_cache-83204968-ba8d-4e40-9f3b-d6d427fc5bb5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 749.300266] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 749.300987] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-12a12a27-0165-4b37-a31a-10fedda67a3c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 749.309461] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3452225-aa3c-4ef0-aabd-24d8b1368e47 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 749.331063] env[63593]: WARNING nova.virt.vmwareapi.driver [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance 83204968-ba8d-4e40-9f3b-d6d427fc5bb5 could not be found. [ 749.331330] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 749.331645] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-06a74bf6-74a2-4a15-b860-e470e0cc0741 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 749.339417] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf427e0f-abb5-4efb-afdc-721c225f945f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 749.361718] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.483s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 749.362234] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 749.364894] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 83204968-ba8d-4e40-9f3b-d6d427fc5bb5 could not be found. [ 749.365099] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 749.365280] env[63593]: INFO nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Took 0.06 seconds to destroy the instance on the hypervisor. [ 749.365561] env[63593]: DEBUG oslo.service.loopingcall [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 749.365769] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 30.736s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.367191] env[63593]: INFO nova.compute.claims [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 749.369486] env[63593]: DEBUG nova.compute.manager [-] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 749.369587] env[63593]: DEBUG nova.network.neutron [-] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 749.384892] env[63593]: DEBUG nova.network.neutron [-] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 749.471457] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 749.563290] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 749.875014] env[63593]: DEBUG nova.compute.utils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 749.878092] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 749.878323] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 749.888213] env[63593]: DEBUG nova.network.neutron [-] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 749.929511] env[63593]: DEBUG nova.policy [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9a66ce1afd7c42d190c257ee11aceb56', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd0d237a61ccd4912be381c4c4902fa6e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 750.066691] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Releasing lock "refresh_cache-40449f7d-3a21-487a-a778-43ccd7b23949" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 750.068148] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 750.068148] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 750.068148] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a6c87af3-3c03-4485-a090-ac53fbe2033f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.078134] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79c033ef-e1d9-43c2-8daf-4a58fa65a719 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.100512] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 40449f7d-3a21-487a-a778-43ccd7b23949 could not be found. [ 750.100749] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 750.100930] env[63593]: INFO nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Took 0.03 seconds to destroy the instance on the hypervisor. [ 750.101198] env[63593]: DEBUG oslo.service.loopingcall [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 750.101423] env[63593]: DEBUG nova.compute.manager [-] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 750.101517] env[63593]: DEBUG nova.network.neutron [-] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 750.118708] env[63593]: DEBUG nova.network.neutron [-] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 750.231710] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Successfully created port: 71903bc8-a64f-4381-815c-0e28674987ea {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 750.258874] env[63593]: DEBUG nova.compute.manager [req-0fd4a1c5-2b68-4e91-9103-ac3e56b8129a req-117a07df-588f-4c76-af0c-50cc54558583 service nova] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Received event network-changed-8482d961-e7b9-4b67-9b93-5a63fdc65d68 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 750.259028] env[63593]: DEBUG nova.compute.manager [req-0fd4a1c5-2b68-4e91-9103-ac3e56b8129a req-117a07df-588f-4c76-af0c-50cc54558583 service nova] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Refreshing instance network info cache due to event network-changed-8482d961-e7b9-4b67-9b93-5a63fdc65d68. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 750.259304] env[63593]: DEBUG oslo_concurrency.lockutils [req-0fd4a1c5-2b68-4e91-9103-ac3e56b8129a req-117a07df-588f-4c76-af0c-50cc54558583 service nova] Acquiring lock "refresh_cache-40449f7d-3a21-487a-a778-43ccd7b23949" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 750.259472] env[63593]: DEBUG oslo_concurrency.lockutils [req-0fd4a1c5-2b68-4e91-9103-ac3e56b8129a req-117a07df-588f-4c76-af0c-50cc54558583 service nova] Acquired lock "refresh_cache-40449f7d-3a21-487a-a778-43ccd7b23949" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 750.259648] env[63593]: DEBUG nova.network.neutron [req-0fd4a1c5-2b68-4e91-9103-ac3e56b8129a req-117a07df-588f-4c76-af0c-50cc54558583 service nova] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Refreshing network info cache for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 750.380802] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 750.391295] env[63593]: INFO nova.compute.manager [-] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Took 1.02 seconds to deallocate network for instance. [ 750.620740] env[63593]: DEBUG nova.network.neutron [-] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 750.718340] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2e33952-70cc-4a3e-bf38-733265d34b88 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.727189] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5633e73-21a4-4cba-98db-3a2dcb469ada {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.757881] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24ec294c-4318-4288-991e-90a3e4d9d658 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.766751] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2d12f3d-be8d-45f0-8d38-0b9ba4cc7341 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.779945] env[63593]: DEBUG nova.compute.provider_tree [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 750.781507] env[63593]: DEBUG nova.network.neutron [req-0fd4a1c5-2b68-4e91-9103-ac3e56b8129a req-117a07df-588f-4c76-af0c-50cc54558583 service nova] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 750.874926] env[63593]: DEBUG nova.network.neutron [req-0fd4a1c5-2b68-4e91-9103-ac3e56b8129a req-117a07df-588f-4c76-af0c-50cc54558583 service nova] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 750.952051] env[63593]: INFO nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Took 0.56 seconds to detach 1 volumes for instance. [ 750.953534] env[63593]: DEBUG nova.compute.claims [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 750.953722] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 751.125982] env[63593]: INFO nova.compute.manager [-] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Took 1.02 seconds to deallocate network for instance. [ 751.131772] env[63593]: DEBUG nova.compute.claims [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 751.132016] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 751.182115] env[63593]: ERROR nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 71903bc8-a64f-4381-815c-0e28674987ea, please check neutron logs for more information. [ 751.182115] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 751.182115] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 751.182115] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 751.182115] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 751.182115] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 751.182115] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 751.182115] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 751.182115] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 751.182115] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 751.182115] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 751.182115] env[63593]: ERROR nova.compute.manager raise self.value [ 751.182115] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 751.182115] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 751.182115] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 751.182115] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 751.182568] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 751.182568] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 751.182568] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 71903bc8-a64f-4381-815c-0e28674987ea, please check neutron logs for more information. [ 751.182568] env[63593]: ERROR nova.compute.manager [ 751.182568] env[63593]: Traceback (most recent call last): [ 751.182568] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 751.182568] env[63593]: listener.cb(fileno) [ 751.182568] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 751.182568] env[63593]: result = function(*args, **kwargs) [ 751.182568] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 751.182568] env[63593]: return func(*args, **kwargs) [ 751.182568] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 751.182568] env[63593]: raise e [ 751.182568] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 751.182568] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 751.182568] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 751.182568] env[63593]: created_port_ids = self._update_ports_for_instance( [ 751.182568] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 751.182568] env[63593]: with excutils.save_and_reraise_exception(): [ 751.182568] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 751.182568] env[63593]: self.force_reraise() [ 751.182568] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 751.182568] env[63593]: raise self.value [ 751.182568] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 751.182568] env[63593]: updated_port = self._update_port( [ 751.182568] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 751.182568] env[63593]: _ensure_no_port_binding_failure(port) [ 751.182568] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 751.182568] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 751.183477] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 71903bc8-a64f-4381-815c-0e28674987ea, please check neutron logs for more information. [ 751.183477] env[63593]: Removing descriptor: 21 [ 751.287019] env[63593]: DEBUG nova.scheduler.client.report [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 751.377161] env[63593]: DEBUG oslo_concurrency.lockutils [req-0fd4a1c5-2b68-4e91-9103-ac3e56b8129a req-117a07df-588f-4c76-af0c-50cc54558583 service nova] Releasing lock "refresh_cache-40449f7d-3a21-487a-a778-43ccd7b23949" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 751.377334] env[63593]: DEBUG nova.compute.manager [req-0fd4a1c5-2b68-4e91-9103-ac3e56b8129a req-117a07df-588f-4c76-af0c-50cc54558583 service nova] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Received event network-vif-deleted-8482d961-e7b9-4b67-9b93-5a63fdc65d68 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 751.391814] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 751.416039] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 751.416687] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 751.416687] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 751.416687] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 751.416957] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 751.416957] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 751.417171] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 751.417406] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 751.417587] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 751.417747] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 751.417912] env[63593]: DEBUG nova.virt.hardware [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 751.418771] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3cf1e2b-1b4a-407a-80f8-56bc60e837d9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.429056] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b87475d4-3624-411c-aa64-35a6220e9e1f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.440604] env[63593]: ERROR nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 71903bc8-a64f-4381-815c-0e28674987ea, please check neutron logs for more information. [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Traceback (most recent call last): [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] yield resources [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self.driver.spawn(context, instance, image_meta, [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self._vmops.spawn(context, instance, image_meta, injected_files, [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] vm_ref = self.build_virtual_machine(instance, [ 751.440604] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] vif_infos = vmwarevif.get_vif_info(self._session, [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] for vif in network_info: [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] return self._sync_wrapper(fn, *args, **kwargs) [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self.wait() [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self[:] = self._gt.wait() [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] return self._exit_event.wait() [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 751.440973] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] current.throw(*self._exc) [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] result = function(*args, **kwargs) [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] return func(*args, **kwargs) [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] raise e [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] nwinfo = self.network_api.allocate_for_instance( [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] created_port_ids = self._update_ports_for_instance( [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] with excutils.save_and_reraise_exception(): [ 751.441297] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self.force_reraise() [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] raise self.value [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] updated_port = self._update_port( [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] _ensure_no_port_binding_failure(port) [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] raise exception.PortBindingFailed(port_id=port['id']) [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] nova.exception.PortBindingFailed: Binding failed for port 71903bc8-a64f-4381-815c-0e28674987ea, please check neutron logs for more information. [ 751.441600] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] [ 751.441600] env[63593]: INFO nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Terminating instance [ 751.442804] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "refresh_cache-70595430-6ab6-49d4-b8fe-131bd44de838" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 751.442962] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquired lock "refresh_cache-70595430-6ab6-49d4-b8fe-131bd44de838" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 751.443136] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 751.790427] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.424s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 751.790987] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 751.793975] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 31.131s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 751.795366] env[63593]: INFO nova.compute.claims [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 751.967541] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 752.057167] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 752.283546] env[63593]: DEBUG nova.compute.manager [req-1c2357c4-99fd-428f-810a-c130978f4196 req-8efba5f8-cefa-45e3-87a5-78eac939ce7f service nova] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Received event network-changed-71903bc8-a64f-4381-815c-0e28674987ea {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 752.283744] env[63593]: DEBUG nova.compute.manager [req-1c2357c4-99fd-428f-810a-c130978f4196 req-8efba5f8-cefa-45e3-87a5-78eac939ce7f service nova] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Refreshing instance network info cache due to event network-changed-71903bc8-a64f-4381-815c-0e28674987ea. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 752.283938] env[63593]: DEBUG oslo_concurrency.lockutils [req-1c2357c4-99fd-428f-810a-c130978f4196 req-8efba5f8-cefa-45e3-87a5-78eac939ce7f service nova] Acquiring lock "refresh_cache-70595430-6ab6-49d4-b8fe-131bd44de838" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 752.299078] env[63593]: DEBUG nova.compute.utils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 752.300371] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 752.300540] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 752.353177] env[63593]: DEBUG nova.policy [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9a66ce1afd7c42d190c257ee11aceb56', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd0d237a61ccd4912be381c4c4902fa6e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 752.564299] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Releasing lock "refresh_cache-70595430-6ab6-49d4-b8fe-131bd44de838" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 752.564777] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 752.564979] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 752.565327] env[63593]: DEBUG oslo_concurrency.lockutils [req-1c2357c4-99fd-428f-810a-c130978f4196 req-8efba5f8-cefa-45e3-87a5-78eac939ce7f service nova] Acquired lock "refresh_cache-70595430-6ab6-49d4-b8fe-131bd44de838" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 752.565499] env[63593]: DEBUG nova.network.neutron [req-1c2357c4-99fd-428f-810a-c130978f4196 req-8efba5f8-cefa-45e3-87a5-78eac939ce7f service nova] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Refreshing network info cache for port 71903bc8-a64f-4381-815c-0e28674987ea {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 752.570619] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e5ba8051-407c-4aee-81a8-e9aa26ecef2b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 752.584434] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04b8a4b8-9ca7-4373-8ba7-eb6ef128d16e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 752.608317] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 70595430-6ab6-49d4-b8fe-131bd44de838 could not be found. [ 752.611262] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 752.611262] env[63593]: INFO nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Took 0.04 seconds to destroy the instance on the hypervisor. [ 752.611262] env[63593]: DEBUG oslo.service.loopingcall [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 752.611262] env[63593]: DEBUG nova.compute.manager [-] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 752.611262] env[63593]: DEBUG nova.network.neutron [-] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 752.627026] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Successfully created port: ede41595-dbc3-4717-a3d6-e73535f68a35 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 752.629584] env[63593]: DEBUG nova.network.neutron [-] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 752.803607] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 753.086990] env[63593]: DEBUG nova.network.neutron [req-1c2357c4-99fd-428f-810a-c130978f4196 req-8efba5f8-cefa-45e3-87a5-78eac939ce7f service nova] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 753.131817] env[63593]: DEBUG nova.network.neutron [-] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 753.138851] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e661495-b142-4767-8342-cfb09d435531 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.147138] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92b55243-3f06-4582-abd5-653c79f132ea {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.176976] env[63593]: DEBUG nova.network.neutron [req-1c2357c4-99fd-428f-810a-c130978f4196 req-8efba5f8-cefa-45e3-87a5-78eac939ce7f service nova] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 753.180741] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3157114b-fa11-4706-ad02-683588deee8c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.186271] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9dba24f5-7859-45d2-a15d-4058ef8bc53a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.201294] env[63593]: DEBUG nova.compute.provider_tree [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 753.552092] env[63593]: ERROR nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ede41595-dbc3-4717-a3d6-e73535f68a35, please check neutron logs for more information. [ 753.552092] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 753.552092] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 753.552092] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 753.552092] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 753.552092] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 753.552092] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 753.552092] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 753.552092] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 753.552092] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 753.552092] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 753.552092] env[63593]: ERROR nova.compute.manager raise self.value [ 753.552092] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 753.552092] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 753.552092] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 753.552092] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 753.552853] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 753.552853] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 753.552853] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ede41595-dbc3-4717-a3d6-e73535f68a35, please check neutron logs for more information. [ 753.552853] env[63593]: ERROR nova.compute.manager [ 753.552853] env[63593]: Traceback (most recent call last): [ 753.552853] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 753.552853] env[63593]: listener.cb(fileno) [ 753.552853] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 753.552853] env[63593]: result = function(*args, **kwargs) [ 753.552853] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 753.552853] env[63593]: return func(*args, **kwargs) [ 753.552853] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 753.552853] env[63593]: raise e [ 753.552853] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 753.552853] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 753.552853] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 753.552853] env[63593]: created_port_ids = self._update_ports_for_instance( [ 753.552853] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 753.552853] env[63593]: with excutils.save_and_reraise_exception(): [ 753.552853] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 753.552853] env[63593]: self.force_reraise() [ 753.552853] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 753.552853] env[63593]: raise self.value [ 753.552853] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 753.552853] env[63593]: updated_port = self._update_port( [ 753.552853] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 753.552853] env[63593]: _ensure_no_port_binding_failure(port) [ 753.552853] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 753.552853] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 753.554029] env[63593]: nova.exception.PortBindingFailed: Binding failed for port ede41595-dbc3-4717-a3d6-e73535f68a35, please check neutron logs for more information. [ 753.554029] env[63593]: Removing descriptor: 21 [ 753.634024] env[63593]: INFO nova.compute.manager [-] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Took 1.02 seconds to deallocate network for instance. [ 753.637104] env[63593]: DEBUG nova.compute.claims [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 753.637104] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 753.682015] env[63593]: DEBUG oslo_concurrency.lockutils [req-1c2357c4-99fd-428f-810a-c130978f4196 req-8efba5f8-cefa-45e3-87a5-78eac939ce7f service nova] Releasing lock "refresh_cache-70595430-6ab6-49d4-b8fe-131bd44de838" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 753.682300] env[63593]: DEBUG nova.compute.manager [req-1c2357c4-99fd-428f-810a-c130978f4196 req-8efba5f8-cefa-45e3-87a5-78eac939ce7f service nova] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Received event network-vif-deleted-71903bc8-a64f-4381-815c-0e28674987ea {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 753.704597] env[63593]: DEBUG nova.scheduler.client.report [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 753.818889] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 753.843718] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 753.843901] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 753.844458] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 753.844458] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 753.844458] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 753.844617] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 753.844699] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 753.844883] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 753.845014] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 753.845192] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 753.845361] env[63593]: DEBUG nova.virt.hardware [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 753.846211] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e7517c1-7050-4acb-bc6e-90e6287300a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.854383] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77238abd-7f74-4851-b38a-38af851dbdc4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.868127] env[63593]: ERROR nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ede41595-dbc3-4717-a3d6-e73535f68a35, please check neutron logs for more information. [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Traceback (most recent call last): [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] yield resources [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self.driver.spawn(context, instance, image_meta, [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self._vmops.spawn(context, instance, image_meta, injected_files, [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] vm_ref = self.build_virtual_machine(instance, [ 753.868127] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] vif_infos = vmwarevif.get_vif_info(self._session, [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] for vif in network_info: [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] return self._sync_wrapper(fn, *args, **kwargs) [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self.wait() [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self[:] = self._gt.wait() [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] return self._exit_event.wait() [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 753.868494] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] current.throw(*self._exc) [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] result = function(*args, **kwargs) [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] return func(*args, **kwargs) [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] raise e [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] nwinfo = self.network_api.allocate_for_instance( [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] created_port_ids = self._update_ports_for_instance( [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] with excutils.save_and_reraise_exception(): [ 753.868890] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self.force_reraise() [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] raise self.value [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] updated_port = self._update_port( [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] _ensure_no_port_binding_failure(port) [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] raise exception.PortBindingFailed(port_id=port['id']) [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] nova.exception.PortBindingFailed: Binding failed for port ede41595-dbc3-4717-a3d6-e73535f68a35, please check neutron logs for more information. [ 753.869374] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] [ 753.869374] env[63593]: INFO nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Terminating instance [ 753.870402] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "refresh_cache-006d2ab8-e38b-4b69-b3b2-115c39e44218" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 753.871395] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquired lock "refresh_cache-006d2ab8-e38b-4b69-b3b2-115c39e44218" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 753.871395] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 754.210798] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.416s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 754.210897] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 754.213801] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.917s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.215255] env[63593]: INFO nova.compute.claims [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 754.305589] env[63593]: DEBUG nova.compute.manager [req-483354f9-234f-409b-b3de-48ddf8b35d0e req-9606ae28-20d8-4fe7-b462-0b37dc3d0f63 service nova] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Received event network-changed-ede41595-dbc3-4717-a3d6-e73535f68a35 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 754.305782] env[63593]: DEBUG nova.compute.manager [req-483354f9-234f-409b-b3de-48ddf8b35d0e req-9606ae28-20d8-4fe7-b462-0b37dc3d0f63 service nova] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Refreshing instance network info cache due to event network-changed-ede41595-dbc3-4717-a3d6-e73535f68a35. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 754.305966] env[63593]: DEBUG oslo_concurrency.lockutils [req-483354f9-234f-409b-b3de-48ddf8b35d0e req-9606ae28-20d8-4fe7-b462-0b37dc3d0f63 service nova] Acquiring lock "refresh_cache-006d2ab8-e38b-4b69-b3b2-115c39e44218" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 754.387540] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 754.477055] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 754.719525] env[63593]: DEBUG nova.compute.utils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 754.722946] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 754.722946] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 754.775901] env[63593]: DEBUG nova.policy [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4dd6b364c50e467783235ed574aa4dae', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '1c5be52fd255496d8526fe4a8b8b6b6e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 754.979910] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Releasing lock "refresh_cache-006d2ab8-e38b-4b69-b3b2-115c39e44218" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 754.981032] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 754.981032] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 754.981111] env[63593]: DEBUG oslo_concurrency.lockutils [req-483354f9-234f-409b-b3de-48ddf8b35d0e req-9606ae28-20d8-4fe7-b462-0b37dc3d0f63 service nova] Acquired lock "refresh_cache-006d2ab8-e38b-4b69-b3b2-115c39e44218" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 754.981298] env[63593]: DEBUG nova.network.neutron [req-483354f9-234f-409b-b3de-48ddf8b35d0e req-9606ae28-20d8-4fe7-b462-0b37dc3d0f63 service nova] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Refreshing network info cache for port ede41595-dbc3-4717-a3d6-e73535f68a35 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 754.982405] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-49ca3f38-a5cf-4b8e-ad26-ea3a5e1785ae {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 754.992665] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c686080a-6569-415f-a8e8-af0a0f6afb29 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.017867] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 006d2ab8-e38b-4b69-b3b2-115c39e44218 could not be found. [ 755.017867] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 755.017867] env[63593]: INFO nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Took 0.04 seconds to destroy the instance on the hypervisor. [ 755.018079] env[63593]: DEBUG oslo.service.loopingcall [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 755.018301] env[63593]: DEBUG nova.compute.manager [-] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 755.018393] env[63593]: DEBUG nova.network.neutron [-] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 755.032837] env[63593]: DEBUG nova.network.neutron [-] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 755.040759] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Successfully created port: 94ea33c3-5b0a-4759-b18b-6ce4367ea133 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 755.224174] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 755.505401] env[63593]: DEBUG nova.network.neutron [req-483354f9-234f-409b-b3de-48ddf8b35d0e req-9606ae28-20d8-4fe7-b462-0b37dc3d0f63 service nova] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 755.537493] env[63593]: DEBUG nova.network.neutron [-] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 755.615620] env[63593]: DEBUG nova.network.neutron [req-483354f9-234f-409b-b3de-48ddf8b35d0e req-9606ae28-20d8-4fe7-b462-0b37dc3d0f63 service nova] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 755.644174] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41f33cbd-28f4-431d-9586-ddd1a6bbfc08 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.652349] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-921e302e-433c-4539-b3fa-2857dd36900c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.687084] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97cde273-44cb-48c0-84f5-04bbe908d019 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.694794] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2fc6cb98-0ee2-4c6a-90d2-9c67f50cab2b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 755.707761] env[63593]: DEBUG nova.compute.provider_tree [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 756.040686] env[63593]: INFO nova.compute.manager [-] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Took 1.02 seconds to deallocate network for instance. [ 756.042819] env[63593]: DEBUG nova.compute.claims [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 756.043121] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 756.119078] env[63593]: DEBUG oslo_concurrency.lockutils [req-483354f9-234f-409b-b3de-48ddf8b35d0e req-9606ae28-20d8-4fe7-b462-0b37dc3d0f63 service nova] Releasing lock "refresh_cache-006d2ab8-e38b-4b69-b3b2-115c39e44218" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 756.119412] env[63593]: DEBUG nova.compute.manager [req-483354f9-234f-409b-b3de-48ddf8b35d0e req-9606ae28-20d8-4fe7-b462-0b37dc3d0f63 service nova] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Received event network-vif-deleted-ede41595-dbc3-4717-a3d6-e73535f68a35 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 756.156757] env[63593]: ERROR nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133, please check neutron logs for more information. [ 756.156757] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 756.156757] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 756.156757] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 756.156757] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 756.156757] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 756.156757] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 756.156757] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 756.156757] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 756.156757] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 756.156757] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 756.156757] env[63593]: ERROR nova.compute.manager raise self.value [ 756.156757] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 756.156757] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 756.156757] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 756.156757] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 756.157352] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 756.157352] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 756.157352] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133, please check neutron logs for more information. [ 756.157352] env[63593]: ERROR nova.compute.manager [ 756.157352] env[63593]: Traceback (most recent call last): [ 756.157352] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 756.157352] env[63593]: listener.cb(fileno) [ 756.157352] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 756.157352] env[63593]: result = function(*args, **kwargs) [ 756.157352] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 756.157352] env[63593]: return func(*args, **kwargs) [ 756.157352] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 756.157352] env[63593]: raise e [ 756.157352] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 756.157352] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 756.157352] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 756.157352] env[63593]: created_port_ids = self._update_ports_for_instance( [ 756.157352] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 756.157352] env[63593]: with excutils.save_and_reraise_exception(): [ 756.157352] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 756.157352] env[63593]: self.force_reraise() [ 756.157352] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 756.157352] env[63593]: raise self.value [ 756.157352] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 756.157352] env[63593]: updated_port = self._update_port( [ 756.157352] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 756.157352] env[63593]: _ensure_no_port_binding_failure(port) [ 756.157352] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 756.157352] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 756.158717] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133, please check neutron logs for more information. [ 756.158717] env[63593]: Removing descriptor: 21 [ 756.210319] env[63593]: DEBUG nova.scheduler.client.report [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 756.236736] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 756.261384] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 756.261627] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 756.261779] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 756.261954] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 756.262114] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 756.262257] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 756.262457] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 756.262611] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 756.262771] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 756.262926] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 756.263106] env[63593]: DEBUG nova.virt.hardware [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 756.263939] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b07c0785-4948-4bdf-89b3-b2152be03eb5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.271745] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1de05875-540f-40ac-8f2b-e63240ee13e9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.286177] env[63593]: ERROR nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133, please check neutron logs for more information. [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Traceback (most recent call last): [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] yield resources [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self.driver.spawn(context, instance, image_meta, [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] vm_ref = self.build_virtual_machine(instance, [ 756.286177] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] vif_infos = vmwarevif.get_vif_info(self._session, [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] for vif in network_info: [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] return self._sync_wrapper(fn, *args, **kwargs) [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self.wait() [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self[:] = self._gt.wait() [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] return self._exit_event.wait() [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 756.286551] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] current.throw(*self._exc) [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] result = function(*args, **kwargs) [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] return func(*args, **kwargs) [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] raise e [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] nwinfo = self.network_api.allocate_for_instance( [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] created_port_ids = self._update_ports_for_instance( [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] with excutils.save_and_reraise_exception(): [ 756.286879] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self.force_reraise() [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] raise self.value [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] updated_port = self._update_port( [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] _ensure_no_port_binding_failure(port) [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] raise exception.PortBindingFailed(port_id=port['id']) [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] nova.exception.PortBindingFailed: Binding failed for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133, please check neutron logs for more information. [ 756.287537] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] [ 756.287537] env[63593]: INFO nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Terminating instance [ 756.288932] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Acquiring lock "refresh_cache-eddf892e-9706-4d33-8241-f8df6c29dee4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 756.289103] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Acquired lock "refresh_cache-eddf892e-9706-4d33-8241-f8df6c29dee4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 756.289266] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 756.331318] env[63593]: DEBUG nova.compute.manager [req-6d267043-4ee3-4db3-8a8a-300cfacd39aa req-a0f60952-b1a8-4b84-9aff-17fa293c097d service nova] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Received event network-changed-94ea33c3-5b0a-4759-b18b-6ce4367ea133 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 756.331512] env[63593]: DEBUG nova.compute.manager [req-6d267043-4ee3-4db3-8a8a-300cfacd39aa req-a0f60952-b1a8-4b84-9aff-17fa293c097d service nova] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Refreshing instance network info cache due to event network-changed-94ea33c3-5b0a-4759-b18b-6ce4367ea133. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 756.331624] env[63593]: DEBUG oslo_concurrency.lockutils [req-6d267043-4ee3-4db3-8a8a-300cfacd39aa req-a0f60952-b1a8-4b84-9aff-17fa293c097d service nova] Acquiring lock "refresh_cache-eddf892e-9706-4d33-8241-f8df6c29dee4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 756.714667] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.501s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 756.715229] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 756.718208] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 32.310s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 756.810877] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 756.905496] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 757.222959] env[63593]: DEBUG nova.compute.utils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 757.228272] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 757.229257] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 757.271668] env[63593]: DEBUG nova.policy [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '1f77880e5be14e78a390da1096ef1159', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '81a35066eae04b09817eecd0f0b651e0', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 757.407901] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Releasing lock "refresh_cache-eddf892e-9706-4d33-8241-f8df6c29dee4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 757.408374] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 757.408556] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 757.408841] env[63593]: DEBUG oslo_concurrency.lockutils [req-6d267043-4ee3-4db3-8a8a-300cfacd39aa req-a0f60952-b1a8-4b84-9aff-17fa293c097d service nova] Acquired lock "refresh_cache-eddf892e-9706-4d33-8241-f8df6c29dee4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 757.408999] env[63593]: DEBUG nova.network.neutron [req-6d267043-4ee3-4db3-8a8a-300cfacd39aa req-a0f60952-b1a8-4b84-9aff-17fa293c097d service nova] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Refreshing network info cache for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 757.410034] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ba6a3f12-26b6-49c9-b652-4daabecc450d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.422987] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6c22c73-bf9d-41ff-8e03-58730565c1b3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.452525] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance eddf892e-9706-4d33-8241-f8df6c29dee4 could not be found. [ 757.452750] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 757.452929] env[63593]: INFO nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Took 0.04 seconds to destroy the instance on the hypervisor. [ 757.453189] env[63593]: DEBUG oslo.service.loopingcall [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 757.455428] env[63593]: DEBUG nova.compute.manager [-] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 757.455529] env[63593]: DEBUG nova.network.neutron [-] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 757.469651] env[63593]: DEBUG nova.network.neutron [-] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 757.592347] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Successfully created port: abd62477-64c2-490d-ba04-a0168d169e67 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 757.608446] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-128c87e4-ded7-41fd-adc9-0e440b441283 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.615834] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4738d535-cf86-4b9c-8f3f-ebf45fdfe341 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.645331] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b1e0962-1e97-4e05-a0e4-bb4c6766bd3d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.652380] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-467d0f69-21dc-4f33-baeb-4317a6b25d81 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.665461] env[63593]: DEBUG nova.compute.provider_tree [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 757.729641] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 757.939254] env[63593]: DEBUG nova.network.neutron [req-6d267043-4ee3-4db3-8a8a-300cfacd39aa req-a0f60952-b1a8-4b84-9aff-17fa293c097d service nova] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 757.972108] env[63593]: DEBUG nova.network.neutron [-] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 758.024080] env[63593]: DEBUG nova.network.neutron [req-6d267043-4ee3-4db3-8a8a-300cfacd39aa req-a0f60952-b1a8-4b84-9aff-17fa293c097d service nova] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 758.169672] env[63593]: DEBUG nova.scheduler.client.report [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 758.362086] env[63593]: DEBUG nova.compute.manager [req-7312aed1-907c-4b99-ac6d-63c462059c04 req-d0a62e6d-35bc-4176-bced-16d1c8553983 service nova] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Received event network-changed-abd62477-64c2-490d-ba04-a0168d169e67 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 758.362367] env[63593]: DEBUG nova.compute.manager [req-7312aed1-907c-4b99-ac6d-63c462059c04 req-d0a62e6d-35bc-4176-bced-16d1c8553983 service nova] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Refreshing instance network info cache due to event network-changed-abd62477-64c2-490d-ba04-a0168d169e67. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 758.362548] env[63593]: DEBUG oslo_concurrency.lockutils [req-7312aed1-907c-4b99-ac6d-63c462059c04 req-d0a62e6d-35bc-4176-bced-16d1c8553983 service nova] Acquiring lock "refresh_cache-c99cbfae-db59-4b84-b8ab-09b84ebe4623" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 758.362689] env[63593]: DEBUG oslo_concurrency.lockutils [req-7312aed1-907c-4b99-ac6d-63c462059c04 req-d0a62e6d-35bc-4176-bced-16d1c8553983 service nova] Acquired lock "refresh_cache-c99cbfae-db59-4b84-b8ab-09b84ebe4623" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 758.362842] env[63593]: DEBUG nova.network.neutron [req-7312aed1-907c-4b99-ac6d-63c462059c04 req-d0a62e6d-35bc-4176-bced-16d1c8553983 service nova] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Refreshing network info cache for port abd62477-64c2-490d-ba04-a0168d169e67 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 758.477324] env[63593]: INFO nova.compute.manager [-] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Took 1.02 seconds to deallocate network for instance. [ 758.479686] env[63593]: DEBUG nova.compute.claims [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 758.479862] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 758.526678] env[63593]: DEBUG oslo_concurrency.lockutils [req-6d267043-4ee3-4db3-8a8a-300cfacd39aa req-a0f60952-b1a8-4b84-9aff-17fa293c097d service nova] Releasing lock "refresh_cache-eddf892e-9706-4d33-8241-f8df6c29dee4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 758.526678] env[63593]: DEBUG nova.compute.manager [req-6d267043-4ee3-4db3-8a8a-300cfacd39aa req-a0f60952-b1a8-4b84-9aff-17fa293c097d service nova] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Received event network-vif-deleted-94ea33c3-5b0a-4759-b18b-6ce4367ea133 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 758.552416] env[63593]: ERROR nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port abd62477-64c2-490d-ba04-a0168d169e67, please check neutron logs for more information. [ 758.552416] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 758.552416] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 758.552416] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 758.552416] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 758.552416] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 758.552416] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 758.552416] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 758.552416] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 758.552416] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 758.552416] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 758.552416] env[63593]: ERROR nova.compute.manager raise self.value [ 758.552416] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 758.552416] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 758.552416] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 758.552416] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 758.552814] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 758.552814] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 758.552814] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port abd62477-64c2-490d-ba04-a0168d169e67, please check neutron logs for more information. [ 758.552814] env[63593]: ERROR nova.compute.manager [ 758.552814] env[63593]: Traceback (most recent call last): [ 758.552814] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 758.552814] env[63593]: listener.cb(fileno) [ 758.552814] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 758.552814] env[63593]: result = function(*args, **kwargs) [ 758.552814] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 758.552814] env[63593]: return func(*args, **kwargs) [ 758.552814] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 758.552814] env[63593]: raise e [ 758.552814] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 758.552814] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 758.552814] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 758.552814] env[63593]: created_port_ids = self._update_ports_for_instance( [ 758.552814] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 758.552814] env[63593]: with excutils.save_and_reraise_exception(): [ 758.552814] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 758.552814] env[63593]: self.force_reraise() [ 758.552814] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 758.552814] env[63593]: raise self.value [ 758.552814] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 758.552814] env[63593]: updated_port = self._update_port( [ 758.552814] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 758.552814] env[63593]: _ensure_no_port_binding_failure(port) [ 758.552814] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 758.552814] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 758.553531] env[63593]: nova.exception.PortBindingFailed: Binding failed for port abd62477-64c2-490d-ba04-a0168d169e67, please check neutron logs for more information. [ 758.553531] env[63593]: Removing descriptor: 21 [ 758.678182] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.960s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 758.679196] env[63593]: ERROR nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e, please check neutron logs for more information. [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Traceback (most recent call last): [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self.driver.spawn(context, instance, image_meta, [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self._vmops.spawn(context, instance, image_meta, injected_files, [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] vm_ref = self.build_virtual_machine(instance, [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] vif_infos = vmwarevif.get_vif_info(self._session, [ 758.679196] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] for vif in network_info: [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] return self._sync_wrapper(fn, *args, **kwargs) [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self.wait() [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self[:] = self._gt.wait() [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] return self._exit_event.wait() [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] result = hub.switch() [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 758.679624] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] return self.greenlet.switch() [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] result = function(*args, **kwargs) [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] return func(*args, **kwargs) [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] raise e [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] nwinfo = self.network_api.allocate_for_instance( [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] created_port_ids = self._update_ports_for_instance( [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] with excutils.save_and_reraise_exception(): [ 758.679993] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] self.force_reraise() [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] raise self.value [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] updated_port = self._update_port( [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] _ensure_no_port_binding_failure(port) [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] raise exception.PortBindingFailed(port_id=port['id']) [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] nova.exception.PortBindingFailed: Binding failed for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e, please check neutron logs for more information. [ 758.680312] env[63593]: ERROR nova.compute.manager [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] [ 758.680577] env[63593]: DEBUG nova.compute.utils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Binding failed for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 758.681170] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 31.324s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 758.682661] env[63593]: INFO nova.compute.claims [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 758.685535] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Build of instance 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff was re-scheduled: Binding failed for port f3bc3473-0998-475d-aa09-c66d9fc6eb0e, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 758.685962] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 758.686226] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "refresh_cache-3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 758.686392] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquired lock "refresh_cache-3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 758.686561] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 758.740652] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 758.764884] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 758.765223] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 758.765379] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 758.765556] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 758.765699] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 758.765844] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 758.766057] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 758.766219] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 758.766382] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 758.766539] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 758.766703] env[63593]: DEBUG nova.virt.hardware [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 758.767701] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f6273a1-8da8-4b2d-987d-959168682f1a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 758.775510] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52618a66-dbbb-4664-a8b2-28974ef8ef7e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 758.789582] env[63593]: ERROR nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port abd62477-64c2-490d-ba04-a0168d169e67, please check neutron logs for more information. [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Traceback (most recent call last): [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] yield resources [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self.driver.spawn(context, instance, image_meta, [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self._vmops.spawn(context, instance, image_meta, injected_files, [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] vm_ref = self.build_virtual_machine(instance, [ 758.789582] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] vif_infos = vmwarevif.get_vif_info(self._session, [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] for vif in network_info: [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] return self._sync_wrapper(fn, *args, **kwargs) [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self.wait() [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self[:] = self._gt.wait() [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] return self._exit_event.wait() [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 758.789864] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] current.throw(*self._exc) [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] result = function(*args, **kwargs) [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] return func(*args, **kwargs) [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] raise e [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] nwinfo = self.network_api.allocate_for_instance( [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] created_port_ids = self._update_ports_for_instance( [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] with excutils.save_and_reraise_exception(): [ 758.790232] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self.force_reraise() [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] raise self.value [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] updated_port = self._update_port( [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] _ensure_no_port_binding_failure(port) [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] raise exception.PortBindingFailed(port_id=port['id']) [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] nova.exception.PortBindingFailed: Binding failed for port abd62477-64c2-490d-ba04-a0168d169e67, please check neutron logs for more information. [ 758.790566] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] [ 758.790566] env[63593]: INFO nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Terminating instance [ 758.791749] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Acquiring lock "refresh_cache-c99cbfae-db59-4b84-b8ab-09b84ebe4623" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 758.877748] env[63593]: DEBUG nova.network.neutron [req-7312aed1-907c-4b99-ac6d-63c462059c04 req-d0a62e6d-35bc-4176-bced-16d1c8553983 service nova] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 758.957022] env[63593]: DEBUG nova.network.neutron [req-7312aed1-907c-4b99-ac6d-63c462059c04 req-d0a62e6d-35bc-4176-bced-16d1c8553983 service nova] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 759.205049] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 759.275705] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 759.459590] env[63593]: DEBUG oslo_concurrency.lockutils [req-7312aed1-907c-4b99-ac6d-63c462059c04 req-d0a62e6d-35bc-4176-bced-16d1c8553983 service nova] Releasing lock "refresh_cache-c99cbfae-db59-4b84-b8ab-09b84ebe4623" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 759.460026] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Acquired lock "refresh_cache-c99cbfae-db59-4b84-b8ab-09b84ebe4623" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 759.460217] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 759.778951] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Releasing lock "refresh_cache-3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 759.779239] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 759.779484] env[63593]: DEBUG nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 759.779671] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 759.798623] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 759.976692] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 760.021434] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5596a3a5-1c1d-4b77-84f7-ca8e53b646c3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.029157] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aed76a63-95bf-4cfd-b07e-a8e5b7089b92 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.062425] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba18a445-6cb6-4e4a-ac0d-9ea3289203fb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.069586] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-caa90f6e-43ff-4174-9fb8-f3211dbd53b8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.082908] env[63593]: DEBUG nova.compute.provider_tree [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 760.106035] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 760.301540] env[63593]: DEBUG nova.network.neutron [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 760.398072] env[63593]: DEBUG nova.compute.manager [req-f7adbac0-811a-4dec-b9f9-ecf4abe8e52a req-eb003f3e-702b-4631-85f1-4ad313da75a5 service nova] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Received event network-vif-deleted-abd62477-64c2-490d-ba04-a0168d169e67 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 760.586301] env[63593]: DEBUG nova.scheduler.client.report [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 760.608534] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Releasing lock "refresh_cache-c99cbfae-db59-4b84-b8ab-09b84ebe4623" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 760.609137] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 760.609417] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 760.609930] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-26cda4aa-ba4b-412d-8611-306e04e0cba6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.618986] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c62d82b4-f35a-4759-a104-d170efdc3a1b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.640179] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c99cbfae-db59-4b84-b8ab-09b84ebe4623 could not be found. [ 760.640391] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 760.640572] env[63593]: INFO nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Took 0.03 seconds to destroy the instance on the hypervisor. [ 760.640802] env[63593]: DEBUG oslo.service.loopingcall [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 760.641016] env[63593]: DEBUG nova.compute.manager [-] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 760.641115] env[63593]: DEBUG nova.network.neutron [-] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 760.655909] env[63593]: DEBUG nova.network.neutron [-] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 760.803996] env[63593]: INFO nova.compute.manager [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff] Took 1.02 seconds to deallocate network for instance. [ 761.092017] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.411s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.092304] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 761.095898] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 31.709s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.096526] env[63593]: INFO nova.compute.claims [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 761.158378] env[63593]: DEBUG nova.network.neutron [-] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 761.601252] env[63593]: DEBUG nova.compute.utils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 761.605043] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 761.605043] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 761.642870] env[63593]: DEBUG nova.policy [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ec75be2e2dd74fa79486d62fb4785b98', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '36ab13ba297f45799e711f2264376b31', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 761.660221] env[63593]: INFO nova.compute.manager [-] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Took 1.02 seconds to deallocate network for instance. [ 761.662360] env[63593]: DEBUG nova.compute.claims [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 761.662541] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.845508] env[63593]: INFO nova.scheduler.client.report [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Deleted allocations for instance 3cab4088-8ab9-4a1c-b84f-5c9e5813fdff [ 761.987578] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Successfully created port: 6d345b99-0643-41ed-ada2-716e98d37f66 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 762.106507] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 762.353800] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a14dc5d-e520-4dfa-8f1c-528814942dcd tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "3cab4088-8ab9-4a1c-b84f-5c9e5813fdff" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 117.302s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 762.529827] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-238dda77-9039-4f3c-bd81-b0d6538a412e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.537766] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a97bba35-f981-4e93-995b-ae699dfc00bd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.572869] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-886a6cf9-f24e-4b78-b0e5-ff78af6e0cde {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.581204] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-edc78d7c-3497-46ee-a2e7-83705da71044 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.596367] env[63593]: DEBUG nova.compute.provider_tree [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 762.615241] env[63593]: INFO nova.virt.block_device [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Booting with volume 77315b59-7256-438d-8da2-509a6992c47d at /dev/sda [ 762.658201] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bea552a6-d8b2-49c0-a97c-4f9cd58e2230 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.667143] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb95871b-468c-408e-b02f-6042263a9489 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.688708] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-006b944c-0eed-4a6e-a9d0-bc1f593e85fd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.696338] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd14c75a-fa04-42c1-ac51-27d17afc0fe1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.718590] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e9a2614-82dd-414b-9018-99e02b0c4276 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.725455] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7485e27b-622b-4729-9062-67026a17c508 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.740963] env[63593]: DEBUG nova.virt.block_device [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Updating existing volume attachment record: 9f2bf0e3-e834-4b9a-90db-a10864520e2c {{(pid=63593) _volume_attach /opt/stack/nova/nova/virt/block_device.py:665}} [ 762.859922] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 763.099453] env[63593]: DEBUG nova.scheduler.client.report [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 763.106695] env[63593]: DEBUG nova.compute.manager [req-d1beefac-2cf1-44cf-b4db-a30f0a2978eb req-befd2496-3b17-48cb-a955-18a688e5dea1 service nova] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Received event network-changed-6d345b99-0643-41ed-ada2-716e98d37f66 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 763.106907] env[63593]: DEBUG nova.compute.manager [req-d1beefac-2cf1-44cf-b4db-a30f0a2978eb req-befd2496-3b17-48cb-a955-18a688e5dea1 service nova] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Refreshing instance network info cache due to event network-changed-6d345b99-0643-41ed-ada2-716e98d37f66. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 763.107290] env[63593]: DEBUG oslo_concurrency.lockutils [req-d1beefac-2cf1-44cf-b4db-a30f0a2978eb req-befd2496-3b17-48cb-a955-18a688e5dea1 service nova] Acquiring lock "refresh_cache-4f1c0deb-5759-44d2-a2ef-f94e1905949d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 763.107651] env[63593]: DEBUG oslo_concurrency.lockutils [req-d1beefac-2cf1-44cf-b4db-a30f0a2978eb req-befd2496-3b17-48cb-a955-18a688e5dea1 service nova] Acquired lock "refresh_cache-4f1c0deb-5759-44d2-a2ef-f94e1905949d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 763.107651] env[63593]: DEBUG nova.network.neutron [req-d1beefac-2cf1-44cf-b4db-a30f0a2978eb req-befd2496-3b17-48cb-a955-18a688e5dea1 service nova] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Refreshing network info cache for port 6d345b99-0643-41ed-ada2-716e98d37f66 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 763.387168] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.402570] env[63593]: ERROR nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6d345b99-0643-41ed-ada2-716e98d37f66, please check neutron logs for more information. [ 763.402570] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 763.402570] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 763.402570] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 763.402570] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 763.402570] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 763.402570] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 763.402570] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 763.402570] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 763.402570] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 763.402570] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 763.402570] env[63593]: ERROR nova.compute.manager raise self.value [ 763.402570] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 763.402570] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 763.402570] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 763.402570] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 763.403160] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 763.403160] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 763.403160] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6d345b99-0643-41ed-ada2-716e98d37f66, please check neutron logs for more information. [ 763.403160] env[63593]: ERROR nova.compute.manager [ 763.403160] env[63593]: Traceback (most recent call last): [ 763.403160] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 763.403160] env[63593]: listener.cb(fileno) [ 763.403160] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 763.403160] env[63593]: result = function(*args, **kwargs) [ 763.403160] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 763.403160] env[63593]: return func(*args, **kwargs) [ 763.403160] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 763.403160] env[63593]: raise e [ 763.403160] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 763.403160] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 763.403160] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 763.403160] env[63593]: created_port_ids = self._update_ports_for_instance( [ 763.403160] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 763.403160] env[63593]: with excutils.save_and_reraise_exception(): [ 763.403160] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 763.403160] env[63593]: self.force_reraise() [ 763.403160] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 763.403160] env[63593]: raise self.value [ 763.403160] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 763.403160] env[63593]: updated_port = self._update_port( [ 763.403160] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 763.403160] env[63593]: _ensure_no_port_binding_failure(port) [ 763.403160] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 763.403160] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 763.403980] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 6d345b99-0643-41ed-ada2-716e98d37f66, please check neutron logs for more information. [ 763.403980] env[63593]: Removing descriptor: 21 [ 763.606632] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.512s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 763.607382] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 763.611030] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.917s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 763.613799] env[63593]: INFO nova.compute.claims [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 763.639043] env[63593]: DEBUG nova.network.neutron [req-d1beefac-2cf1-44cf-b4db-a30f0a2978eb req-befd2496-3b17-48cb-a955-18a688e5dea1 service nova] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 763.881138] env[63593]: DEBUG nova.network.neutron [req-d1beefac-2cf1-44cf-b4db-a30f0a2978eb req-befd2496-3b17-48cb-a955-18a688e5dea1 service nova] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 764.117405] env[63593]: DEBUG nova.compute.utils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 764.118861] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 764.119040] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 764.164625] env[63593]: DEBUG nova.policy [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea62c292bb2b4922a560a64191173304', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '83b28b01efea4d77b238410c62cea17d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 764.383495] env[63593]: DEBUG oslo_concurrency.lockutils [req-d1beefac-2cf1-44cf-b4db-a30f0a2978eb req-befd2496-3b17-48cb-a955-18a688e5dea1 service nova] Releasing lock "refresh_cache-4f1c0deb-5759-44d2-a2ef-f94e1905949d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 764.521495] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Successfully created port: 13980774-79a8-45fe-a600-24f8ff744044 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 764.623708] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 764.855971] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 764.856599] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.002s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 764.863994] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 764.864807] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 764.866094] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 764.866094] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 764.866094] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 764.866094] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 764.866094] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 764.866483] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 764.866483] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 764.866796] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 764.866996] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 764.867246] env[63593]: DEBUG nova.virt.hardware [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 764.868380] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e298eaee-492e-43e2-babc-5c04b310716b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.879325] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6c4f05f-bedd-48d8-80a4-1d38bf1d40c8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.896013] env[63593]: ERROR nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6d345b99-0643-41ed-ada2-716e98d37f66, please check neutron logs for more information. [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Traceback (most recent call last): [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] yield resources [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self.driver.spawn(context, instance, image_meta, [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] vm_ref = self.build_virtual_machine(instance, [ 764.896013] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] vif_infos = vmwarevif.get_vif_info(self._session, [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] for vif in network_info: [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] return self._sync_wrapper(fn, *args, **kwargs) [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self.wait() [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self[:] = self._gt.wait() [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] return self._exit_event.wait() [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 764.896366] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] current.throw(*self._exc) [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] result = function(*args, **kwargs) [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] return func(*args, **kwargs) [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] raise e [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] nwinfo = self.network_api.allocate_for_instance( [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] created_port_ids = self._update_ports_for_instance( [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] with excutils.save_and_reraise_exception(): [ 764.896746] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self.force_reraise() [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] raise self.value [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] updated_port = self._update_port( [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] _ensure_no_port_binding_failure(port) [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] raise exception.PortBindingFailed(port_id=port['id']) [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] nova.exception.PortBindingFailed: Binding failed for port 6d345b99-0643-41ed-ada2-716e98d37f66, please check neutron logs for more information. [ 764.897168] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] [ 764.897168] env[63593]: INFO nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Terminating instance [ 764.900342] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Acquiring lock "refresh_cache-4f1c0deb-5759-44d2-a2ef-f94e1905949d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 764.900506] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Acquired lock "refresh_cache-4f1c0deb-5759-44d2-a2ef-f94e1905949d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 764.900669] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 765.002376] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b5d629a-ed5a-46d7-b46c-b0756c2113d8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.015248] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18ef6f76-52da-44c6-9d86-9c93a9410f03 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.046985] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd336bbe-aafe-425e-94ee-42837919727b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.054698] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-970cf3ce-1b50-4b61-b8c8-d3d7cb5cfaee {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.068278] env[63593]: DEBUG nova.compute.provider_tree [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 765.132396] env[63593]: DEBUG nova.compute.manager [req-e97241ce-6e7e-4584-ad8a-d5e8a79c1577 req-8962bce7-d0d8-47f4-84aa-78e4469255c9 service nova] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Received event network-vif-deleted-6d345b99-0643-41ed-ada2-716e98d37f66 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 765.423500] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 765.466019] env[63593]: ERROR nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 13980774-79a8-45fe-a600-24f8ff744044, please check neutron logs for more information. [ 765.466019] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 765.466019] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 765.466019] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 765.466019] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 765.466019] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 765.466019] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 765.466019] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 765.466019] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 765.466019] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 765.466019] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 765.466019] env[63593]: ERROR nova.compute.manager raise self.value [ 765.466019] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 765.466019] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 765.466019] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 765.466019] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 765.466475] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 765.466475] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 765.466475] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 13980774-79a8-45fe-a600-24f8ff744044, please check neutron logs for more information. [ 765.466475] env[63593]: ERROR nova.compute.manager [ 765.466475] env[63593]: Traceback (most recent call last): [ 765.466475] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 765.466475] env[63593]: listener.cb(fileno) [ 765.466475] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 765.466475] env[63593]: result = function(*args, **kwargs) [ 765.466475] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 765.466475] env[63593]: return func(*args, **kwargs) [ 765.466475] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 765.466475] env[63593]: raise e [ 765.466475] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 765.466475] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 765.466475] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 765.466475] env[63593]: created_port_ids = self._update_ports_for_instance( [ 765.466475] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 765.466475] env[63593]: with excutils.save_and_reraise_exception(): [ 765.466475] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 765.466475] env[63593]: self.force_reraise() [ 765.466475] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 765.466475] env[63593]: raise self.value [ 765.466475] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 765.466475] env[63593]: updated_port = self._update_port( [ 765.466475] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 765.466475] env[63593]: _ensure_no_port_binding_failure(port) [ 765.466475] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 765.466475] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 765.467245] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 13980774-79a8-45fe-a600-24f8ff744044, please check neutron logs for more information. [ 765.467245] env[63593]: Removing descriptor: 21 [ 765.519263] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 765.570841] env[63593]: DEBUG nova.scheduler.client.report [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 765.639128] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 765.663532] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 765.663785] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 765.663963] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 765.664132] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 765.664281] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 765.664423] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 765.664620] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 765.664793] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 765.664982] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 765.665185] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 765.665366] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 765.666217] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50a2e475-852d-4183-a2ab-84cd5447ec31 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.674640] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dfc83ea-77db-40c6-a56d-433b4d1c8f92 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.689222] env[63593]: ERROR nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 13980774-79a8-45fe-a600-24f8ff744044, please check neutron logs for more information. [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Traceback (most recent call last): [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] yield resources [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self.driver.spawn(context, instance, image_meta, [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self._vmops.spawn(context, instance, image_meta, injected_files, [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] vm_ref = self.build_virtual_machine(instance, [ 765.689222] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] vif_infos = vmwarevif.get_vif_info(self._session, [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] for vif in network_info: [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] return self._sync_wrapper(fn, *args, **kwargs) [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self.wait() [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self[:] = self._gt.wait() [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] return self._exit_event.wait() [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 765.689536] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] current.throw(*self._exc) [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] result = function(*args, **kwargs) [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] return func(*args, **kwargs) [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] raise e [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] nwinfo = self.network_api.allocate_for_instance( [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] created_port_ids = self._update_ports_for_instance( [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] with excutils.save_and_reraise_exception(): [ 765.689840] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self.force_reraise() [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] raise self.value [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] updated_port = self._update_port( [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] _ensure_no_port_binding_failure(port) [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] raise exception.PortBindingFailed(port_id=port['id']) [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] nova.exception.PortBindingFailed: Binding failed for port 13980774-79a8-45fe-a600-24f8ff744044, please check neutron logs for more information. [ 765.690170] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] [ 765.690170] env[63593]: INFO nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Terminating instance [ 765.691683] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "refresh_cache-4da891e9-39e0-4739-b1b9-fb438c1111ab" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 765.691839] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "refresh_cache-4da891e9-39e0-4739-b1b9-fb438c1111ab" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 765.692011] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 766.024261] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Releasing lock "refresh_cache-4f1c0deb-5759-44d2-a2ef-f94e1905949d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 766.024628] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 766.024990] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2a3274b8-01e7-4985-a9ac-39b1fddeaf71 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.034359] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d5ae980-119d-4151-9661-7d1828f9c605 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.056906] env[63593]: WARNING nova.virt.vmwareapi.driver [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance 4f1c0deb-5759-44d2-a2ef-f94e1905949d could not be found. [ 766.057163] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 766.057438] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1be9f933-3e5a-41c1-be09-692578a651c2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.064711] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aaf611ee-20f2-440c-ae51-46116815c679 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.076052] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.465s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 766.076380] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 766.078901] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.125s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 766.092725] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4f1c0deb-5759-44d2-a2ef-f94e1905949d could not be found. [ 766.092843] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 766.092970] env[63593]: INFO nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Took 0.07 seconds to destroy the instance on the hypervisor. [ 766.093239] env[63593]: DEBUG oslo.service.loopingcall [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 766.093459] env[63593]: DEBUG nova.compute.manager [-] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 766.093555] env[63593]: DEBUG nova.network.neutron [-] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 766.110132] env[63593]: DEBUG nova.network.neutron [-] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 766.207599] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 766.285383] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 766.583717] env[63593]: DEBUG nova.compute.utils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 766.585507] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 766.585671] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 766.612342] env[63593]: DEBUG nova.network.neutron [-] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 766.647293] env[63593]: DEBUG nova.policy [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea62c292bb2b4922a560a64191173304', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '83b28b01efea4d77b238410c62cea17d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 766.787755] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "refresh_cache-4da891e9-39e0-4739-b1b9-fb438c1111ab" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 766.788405] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 766.788722] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 766.789156] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2d8b59e1-b063-4680-8624-1b534bed44db {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.808493] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3754959-5f99-4ac4-8c22-f907715306c0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.845729] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4da891e9-39e0-4739-b1b9-fb438c1111ab could not be found. [ 766.846099] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 766.846402] env[63593]: INFO nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Took 0.06 seconds to destroy the instance on the hypervisor. [ 766.846833] env[63593]: DEBUG oslo.service.loopingcall [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 766.850766] env[63593]: DEBUG nova.compute.manager [-] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 766.850932] env[63593]: DEBUG nova.network.neutron [-] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 766.870212] env[63593]: DEBUG nova.network.neutron [-] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 766.982474] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Successfully created port: d1bc7083-f53f-4bee-8d7e-39e81796668a {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 767.066199] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b852de7-51e4-4635-b950-59f562c289d8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.074207] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32bca8c8-104d-422c-b8e2-afb1729b4e0a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.107213] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 767.114030] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c78e4288-6d06-4ac3-ab62-b0dd113dc72a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.116044] env[63593]: INFO nova.compute.manager [-] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Took 1.02 seconds to deallocate network for instance. [ 767.125280] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e10bd5b0-ebf9-4a99-9ebf-0e43b3f4b6bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.141923] env[63593]: DEBUG nova.compute.provider_tree [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 767.170275] env[63593]: DEBUG nova.compute.manager [req-b9ef4fe5-ca70-4a10-8f43-08ed50323544 req-1ad9db1d-94e2-4dc9-a19a-f9d975df7406 service nova] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Received event network-changed-13980774-79a8-45fe-a600-24f8ff744044 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 767.170591] env[63593]: DEBUG nova.compute.manager [req-b9ef4fe5-ca70-4a10-8f43-08ed50323544 req-1ad9db1d-94e2-4dc9-a19a-f9d975df7406 service nova] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Refreshing instance network info cache due to event network-changed-13980774-79a8-45fe-a600-24f8ff744044. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 767.170922] env[63593]: DEBUG oslo_concurrency.lockutils [req-b9ef4fe5-ca70-4a10-8f43-08ed50323544 req-1ad9db1d-94e2-4dc9-a19a-f9d975df7406 service nova] Acquiring lock "refresh_cache-4da891e9-39e0-4739-b1b9-fb438c1111ab" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 767.171177] env[63593]: DEBUG oslo_concurrency.lockutils [req-b9ef4fe5-ca70-4a10-8f43-08ed50323544 req-1ad9db1d-94e2-4dc9-a19a-f9d975df7406 service nova] Acquired lock "refresh_cache-4da891e9-39e0-4739-b1b9-fb438c1111ab" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 767.171480] env[63593]: DEBUG nova.network.neutron [req-b9ef4fe5-ca70-4a10-8f43-08ed50323544 req-1ad9db1d-94e2-4dc9-a19a-f9d975df7406 service nova] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Refreshing network info cache for port 13980774-79a8-45fe-a600-24f8ff744044 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 767.372766] env[63593]: DEBUG nova.network.neutron [-] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.645025] env[63593]: DEBUG nova.scheduler.client.report [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 767.695111] env[63593]: DEBUG nova.network.neutron [req-b9ef4fe5-ca70-4a10-8f43-08ed50323544 req-1ad9db1d-94e2-4dc9-a19a-f9d975df7406 service nova] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 767.705992] env[63593]: INFO nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Took 0.59 seconds to detach 1 volumes for instance. [ 767.705992] env[63593]: DEBUG nova.compute.claims [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 767.705992] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.786672] env[63593]: DEBUG nova.compute.manager [req-a8617575-5673-40f9-9192-c7e5c71cfaab req-e239573e-9e37-462d-ab88-e93142ee757a service nova] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Received event network-changed-d1bc7083-f53f-4bee-8d7e-39e81796668a {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 767.786860] env[63593]: DEBUG nova.compute.manager [req-a8617575-5673-40f9-9192-c7e5c71cfaab req-e239573e-9e37-462d-ab88-e93142ee757a service nova] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Refreshing instance network info cache due to event network-changed-d1bc7083-f53f-4bee-8d7e-39e81796668a. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 767.787947] env[63593]: DEBUG oslo_concurrency.lockutils [req-a8617575-5673-40f9-9192-c7e5c71cfaab req-e239573e-9e37-462d-ab88-e93142ee757a service nova] Acquiring lock "refresh_cache-788e5910-c189-4d13-a2fa-e5d6e915e95d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 767.788205] env[63593]: DEBUG oslo_concurrency.lockutils [req-a8617575-5673-40f9-9192-c7e5c71cfaab req-e239573e-9e37-462d-ab88-e93142ee757a service nova] Acquired lock "refresh_cache-788e5910-c189-4d13-a2fa-e5d6e915e95d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 767.788418] env[63593]: DEBUG nova.network.neutron [req-a8617575-5673-40f9-9192-c7e5c71cfaab req-e239573e-9e37-462d-ab88-e93142ee757a service nova] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Refreshing network info cache for port d1bc7083-f53f-4bee-8d7e-39e81796668a {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 767.791693] env[63593]: DEBUG nova.network.neutron [req-b9ef4fe5-ca70-4a10-8f43-08ed50323544 req-1ad9db1d-94e2-4dc9-a19a-f9d975df7406 service nova] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.875645] env[63593]: INFO nova.compute.manager [-] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Took 1.02 seconds to deallocate network for instance. [ 767.880456] env[63593]: DEBUG nova.compute.claims [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 767.881304] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 768.004260] env[63593]: ERROR nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d1bc7083-f53f-4bee-8d7e-39e81796668a, please check neutron logs for more information. [ 768.004260] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 768.004260] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 768.004260] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 768.004260] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 768.004260] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 768.004260] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 768.004260] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 768.004260] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 768.004260] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 768.004260] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 768.004260] env[63593]: ERROR nova.compute.manager raise self.value [ 768.004260] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 768.004260] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 768.004260] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 768.004260] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 768.004938] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 768.004938] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 768.004938] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d1bc7083-f53f-4bee-8d7e-39e81796668a, please check neutron logs for more information. [ 768.004938] env[63593]: ERROR nova.compute.manager [ 768.004938] env[63593]: Traceback (most recent call last): [ 768.004938] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 768.004938] env[63593]: listener.cb(fileno) [ 768.004938] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 768.004938] env[63593]: result = function(*args, **kwargs) [ 768.004938] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 768.004938] env[63593]: return func(*args, **kwargs) [ 768.004938] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 768.004938] env[63593]: raise e [ 768.004938] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 768.004938] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 768.004938] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 768.004938] env[63593]: created_port_ids = self._update_ports_for_instance( [ 768.004938] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 768.004938] env[63593]: with excutils.save_and_reraise_exception(): [ 768.004938] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 768.004938] env[63593]: self.force_reraise() [ 768.004938] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 768.004938] env[63593]: raise self.value [ 768.004938] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 768.004938] env[63593]: updated_port = self._update_port( [ 768.004938] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 768.004938] env[63593]: _ensure_no_port_binding_failure(port) [ 768.004938] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 768.004938] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 768.006068] env[63593]: nova.exception.PortBindingFailed: Binding failed for port d1bc7083-f53f-4bee-8d7e-39e81796668a, please check neutron logs for more information. [ 768.006068] env[63593]: Removing descriptor: 19 [ 768.123559] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 768.147920] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 768.148311] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 768.148518] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 768.148721] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 768.148850] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 768.149026] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 768.149232] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 768.149530] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 768.149780] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 768.149961] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 768.150164] env[63593]: DEBUG nova.virt.hardware [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 768.150891] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.072s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 768.151484] env[63593]: ERROR nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0108f921-dc23-4118-ab51-5b67dd87d26f, please check neutron logs for more information. [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Traceback (most recent call last): [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self.driver.spawn(context, instance, image_meta, [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] vm_ref = self.build_virtual_machine(instance, [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] vif_infos = vmwarevif.get_vif_info(self._session, [ 768.151484] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] for vif in network_info: [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] return self._sync_wrapper(fn, *args, **kwargs) [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self.wait() [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self[:] = self._gt.wait() [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] return self._exit_event.wait() [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] current.throw(*self._exc) [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 768.151775] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] result = function(*args, **kwargs) [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] return func(*args, **kwargs) [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] raise e [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] nwinfo = self.network_api.allocate_for_instance( [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] created_port_ids = self._update_ports_for_instance( [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] with excutils.save_and_reraise_exception(): [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] self.force_reraise() [ 768.152097] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 768.152407] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] raise self.value [ 768.152407] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 768.152407] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] updated_port = self._update_port( [ 768.152407] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 768.152407] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] _ensure_no_port_binding_failure(port) [ 768.152407] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 768.152407] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] raise exception.PortBindingFailed(port_id=port['id']) [ 768.152407] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] nova.exception.PortBindingFailed: Binding failed for port 0108f921-dc23-4118-ab51-5b67dd87d26f, please check neutron logs for more information. [ 768.152407] env[63593]: ERROR nova.compute.manager [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] [ 768.152407] env[63593]: DEBUG nova.compute.utils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Binding failed for port 0108f921-dc23-4118-ab51-5b67dd87d26f, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 768.153789] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7eb294d5-355b-49cb-830c-283f1240b465 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.157222] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Build of instance 83204968-ba8d-4e40-9f3b-d6d427fc5bb5 was re-scheduled: Binding failed for port 0108f921-dc23-4118-ab51-5b67dd87d26f, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 768.157649] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 768.157873] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Acquiring lock "refresh_cache-83204968-ba8d-4e40-9f3b-d6d427fc5bb5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 768.158022] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Acquired lock "refresh_cache-83204968-ba8d-4e40-9f3b-d6d427fc5bb5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 768.158178] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 768.159167] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.027s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 768.167879] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2666538d-f208-4775-88c4-71b788ca5400 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.183928] env[63593]: ERROR nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d1bc7083-f53f-4bee-8d7e-39e81796668a, please check neutron logs for more information. [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Traceback (most recent call last): [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] yield resources [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self.driver.spawn(context, instance, image_meta, [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] vm_ref = self.build_virtual_machine(instance, [ 768.183928] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] vif_infos = vmwarevif.get_vif_info(self._session, [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] for vif in network_info: [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] return self._sync_wrapper(fn, *args, **kwargs) [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self.wait() [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self[:] = self._gt.wait() [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] return self._exit_event.wait() [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 768.184379] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] current.throw(*self._exc) [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] result = function(*args, **kwargs) [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] return func(*args, **kwargs) [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] raise e [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] nwinfo = self.network_api.allocate_for_instance( [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] created_port_ids = self._update_ports_for_instance( [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] with excutils.save_and_reraise_exception(): [ 768.184752] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self.force_reraise() [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] raise self.value [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] updated_port = self._update_port( [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] _ensure_no_port_binding_failure(port) [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] raise exception.PortBindingFailed(port_id=port['id']) [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] nova.exception.PortBindingFailed: Binding failed for port d1bc7083-f53f-4bee-8d7e-39e81796668a, please check neutron logs for more information. [ 768.185351] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] [ 768.185351] env[63593]: INFO nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Terminating instance [ 768.186464] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "refresh_cache-788e5910-c189-4d13-a2fa-e5d6e915e95d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 768.294717] env[63593]: DEBUG oslo_concurrency.lockutils [req-b9ef4fe5-ca70-4a10-8f43-08ed50323544 req-1ad9db1d-94e2-4dc9-a19a-f9d975df7406 service nova] Releasing lock "refresh_cache-4da891e9-39e0-4739-b1b9-fb438c1111ab" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 768.295089] env[63593]: DEBUG nova.compute.manager [req-b9ef4fe5-ca70-4a10-8f43-08ed50323544 req-1ad9db1d-94e2-4dc9-a19a-f9d975df7406 service nova] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Received event network-vif-deleted-13980774-79a8-45fe-a600-24f8ff744044 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 768.310085] env[63593]: DEBUG nova.network.neutron [req-a8617575-5673-40f9-9192-c7e5c71cfaab req-e239573e-9e37-462d-ab88-e93142ee757a service nova] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 768.399612] env[63593]: DEBUG nova.network.neutron [req-a8617575-5673-40f9-9192-c7e5c71cfaab req-e239573e-9e37-462d-ab88-e93142ee757a service nova] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 768.684027] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 768.774093] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 768.902086] env[63593]: DEBUG oslo_concurrency.lockutils [req-a8617575-5673-40f9-9192-c7e5c71cfaab req-e239573e-9e37-462d-ab88-e93142ee757a service nova] Releasing lock "refresh_cache-788e5910-c189-4d13-a2fa-e5d6e915e95d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 768.902549] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "refresh_cache-788e5910-c189-4d13-a2fa-e5d6e915e95d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 768.902741] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 768.975826] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18de20bc-7e64-464b-a140-3759f1c778e5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.983114] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13a0492d-0674-43a9-b96f-de031c2bcd9c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.012059] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43cf1bce-8eaa-4fde-81c9-4cd9be294c94 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.019581] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-103c75f1-9329-42db-a190-ad82b438151c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.034020] env[63593]: DEBUG nova.compute.provider_tree [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 769.278023] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Releasing lock "refresh_cache-83204968-ba8d-4e40-9f3b-d6d427fc5bb5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 769.278449] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 769.278666] env[63593]: DEBUG nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 769.278835] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 769.294701] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 769.418849] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 769.486260] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 769.536689] env[63593]: DEBUG nova.scheduler.client.report [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 769.797431] env[63593]: DEBUG nova.network.neutron [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 769.818421] env[63593]: DEBUG nova.compute.manager [req-99b67de2-34a0-461b-96b5-84589792f22b req-92039d4b-9a79-4d6d-b59d-adc884aebb48 service nova] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Received event network-vif-deleted-d1bc7083-f53f-4bee-8d7e-39e81796668a {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 769.989071] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "refresh_cache-788e5910-c189-4d13-a2fa-e5d6e915e95d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 769.990023] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 769.990023] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 769.990242] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ec58e8c7-7097-4eef-b0bb-17ee37898220 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.999226] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06b1284f-e7a0-4075-963f-24f9c9ef20b4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.020095] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 788e5910-c189-4d13-a2fa-e5d6e915e95d could not be found. [ 770.020296] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 770.020477] env[63593]: INFO nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 770.020703] env[63593]: DEBUG oslo.service.loopingcall [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 770.020922] env[63593]: DEBUG nova.compute.manager [-] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 770.021025] env[63593]: DEBUG nova.network.neutron [-] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 770.038263] env[63593]: DEBUG nova.network.neutron [-] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 770.041965] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.883s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 770.042566] env[63593]: ERROR nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68, please check neutron logs for more information. [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Traceback (most recent call last): [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self.driver.spawn(context, instance, image_meta, [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self._vmops.spawn(context, instance, image_meta, injected_files, [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] vm_ref = self.build_virtual_machine(instance, [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] vif_infos = vmwarevif.get_vif_info(self._session, [ 770.042566] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] for vif in network_info: [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] return self._sync_wrapper(fn, *args, **kwargs) [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self.wait() [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self[:] = self._gt.wait() [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] return self._exit_event.wait() [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] current.throw(*self._exc) [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 770.042903] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] result = function(*args, **kwargs) [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] return func(*args, **kwargs) [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] raise e [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] nwinfo = self.network_api.allocate_for_instance( [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] created_port_ids = self._update_ports_for_instance( [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] with excutils.save_and_reraise_exception(): [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] self.force_reraise() [ 770.043349] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 770.043722] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] raise self.value [ 770.043722] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 770.043722] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] updated_port = self._update_port( [ 770.043722] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 770.043722] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] _ensure_no_port_binding_failure(port) [ 770.043722] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 770.043722] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] raise exception.PortBindingFailed(port_id=port['id']) [ 770.043722] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] nova.exception.PortBindingFailed: Binding failed for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68, please check neutron logs for more information. [ 770.043722] env[63593]: ERROR nova.compute.manager [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] [ 770.043722] env[63593]: DEBUG nova.compute.utils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Binding failed for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 770.044707] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Build of instance 40449f7d-3a21-487a-a778-43ccd7b23949 was re-scheduled: Binding failed for port 8482d961-e7b9-4b67-9b93-5a63fdc65d68, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 770.045373] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 770.045614] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Acquiring lock "refresh_cache-40449f7d-3a21-487a-a778-43ccd7b23949" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 770.045771] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Acquired lock "refresh_cache-40449f7d-3a21-487a-a778-43ccd7b23949" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 770.045916] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 770.047229] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.410s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 770.300320] env[63593]: INFO nova.compute.manager [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] [instance: 83204968-ba8d-4e40-9f3b-d6d427fc5bb5] Took 1.02 seconds to deallocate network for instance. [ 770.541344] env[63593]: DEBUG nova.network.neutron [-] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 770.570457] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 770.655251] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 770.877955] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f879b15e-c0df-4561-a977-65e35a51ebc6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.886419] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc69ff05-81cd-40f8-9bfb-901f4962a9c6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.915513] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d65d4eb-974a-4bbf-ae65-4c9909f72a87 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.922262] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c3260a2-4b4f-4335-95b2-6f8160d69366 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.935129] env[63593]: DEBUG nova.compute.provider_tree [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 771.045336] env[63593]: INFO nova.compute.manager [-] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Took 1.02 seconds to deallocate network for instance. [ 771.048228] env[63593]: DEBUG nova.compute.claims [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 771.048228] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 771.157699] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Releasing lock "refresh_cache-40449f7d-3a21-487a-a778-43ccd7b23949" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 771.157986] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 771.158193] env[63593]: DEBUG nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 771.158362] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 771.173882] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 771.327839] env[63593]: INFO nova.scheduler.client.report [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Deleted allocations for instance 83204968-ba8d-4e40-9f3b-d6d427fc5bb5 [ 771.438435] env[63593]: DEBUG nova.scheduler.client.report [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 771.676567] env[63593]: DEBUG nova.network.neutron [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 771.836164] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db62ca53-b13b-4a87-88e8-0817a3d355de tempest-ServersTestBootFromVolume-982297717 tempest-ServersTestBootFromVolume-982297717-project-member] Lock "83204968-ba8d-4e40-9f3b-d6d427fc5bb5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 125.908s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 771.945263] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.898s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 771.945891] env[63593]: ERROR nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 71903bc8-a64f-4381-815c-0e28674987ea, please check neutron logs for more information. [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Traceback (most recent call last): [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self.driver.spawn(context, instance, image_meta, [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self._vmops.spawn(context, instance, image_meta, injected_files, [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] vm_ref = self.build_virtual_machine(instance, [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] vif_infos = vmwarevif.get_vif_info(self._session, [ 771.945891] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] for vif in network_info: [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] return self._sync_wrapper(fn, *args, **kwargs) [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self.wait() [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self[:] = self._gt.wait() [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] return self._exit_event.wait() [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] current.throw(*self._exc) [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 771.946248] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] result = function(*args, **kwargs) [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] return func(*args, **kwargs) [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] raise e [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] nwinfo = self.network_api.allocate_for_instance( [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] created_port_ids = self._update_ports_for_instance( [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] with excutils.save_and_reraise_exception(): [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] self.force_reraise() [ 771.946615] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 771.946976] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] raise self.value [ 771.946976] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 771.946976] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] updated_port = self._update_port( [ 771.946976] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 771.946976] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] _ensure_no_port_binding_failure(port) [ 771.946976] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 771.946976] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] raise exception.PortBindingFailed(port_id=port['id']) [ 771.946976] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] nova.exception.PortBindingFailed: Binding failed for port 71903bc8-a64f-4381-815c-0e28674987ea, please check neutron logs for more information. [ 771.946976] env[63593]: ERROR nova.compute.manager [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] [ 771.946976] env[63593]: DEBUG nova.compute.utils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Binding failed for port 71903bc8-a64f-4381-815c-0e28674987ea, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 771.947832] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.905s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 771.951304] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Build of instance 70595430-6ab6-49d4-b8fe-131bd44de838 was re-scheduled: Binding failed for port 71903bc8-a64f-4381-815c-0e28674987ea, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 771.951780] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 771.952029] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "refresh_cache-70595430-6ab6-49d4-b8fe-131bd44de838" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 771.952223] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquired lock "refresh_cache-70595430-6ab6-49d4-b8fe-131bd44de838" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 771.952435] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 772.179349] env[63593]: INFO nova.compute.manager [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] [instance: 40449f7d-3a21-487a-a778-43ccd7b23949] Took 1.02 seconds to deallocate network for instance. [ 772.339270] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 772.474757] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 772.597511] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 772.823091] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8cd25ca-edf6-479c-852f-17e8cf1e49bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.830354] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85352dd2-d343-42c8-bab3-aa92a7774ecd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.865932] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a22b696-0384-4051-a5d3-8157bd05ce27 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.874498] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08f8d770-7642-45a1-b601-303829d19c1a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.889213] env[63593]: DEBUG nova.compute.provider_tree [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 772.889569] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 773.100901] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Releasing lock "refresh_cache-70595430-6ab6-49d4-b8fe-131bd44de838" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 773.101159] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 773.101336] env[63593]: DEBUG nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 773.101497] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 773.120688] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 773.210881] env[63593]: INFO nova.scheduler.client.report [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Deleted allocations for instance 40449f7d-3a21-487a-a778-43ccd7b23949 [ 773.391479] env[63593]: DEBUG nova.scheduler.client.report [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 773.626280] env[63593]: DEBUG nova.network.neutron [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 773.719368] env[63593]: DEBUG oslo_concurrency.lockutils [None req-6c27b9bb-24d8-4a4d-8268-c2c7de95ce11 tempest-ServerAddressesTestJSON-1531298909 tempest-ServerAddressesTestJSON-1531298909-project-member] Lock "40449f7d-3a21-487a-a778-43ccd7b23949" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 125.738s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.897394] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.949s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.898047] env[63593]: ERROR nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ede41595-dbc3-4717-a3d6-e73535f68a35, please check neutron logs for more information. [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Traceback (most recent call last): [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self.driver.spawn(context, instance, image_meta, [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self._vmops.spawn(context, instance, image_meta, injected_files, [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] vm_ref = self.build_virtual_machine(instance, [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] vif_infos = vmwarevif.get_vif_info(self._session, [ 773.898047] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] for vif in network_info: [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] return self._sync_wrapper(fn, *args, **kwargs) [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self.wait() [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self[:] = self._gt.wait() [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] return self._exit_event.wait() [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] current.throw(*self._exc) [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 773.898381] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] result = function(*args, **kwargs) [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] return func(*args, **kwargs) [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] raise e [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] nwinfo = self.network_api.allocate_for_instance( [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] created_port_ids = self._update_ports_for_instance( [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] with excutils.save_and_reraise_exception(): [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] self.force_reraise() [ 773.898755] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 773.899134] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] raise self.value [ 773.899134] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 773.899134] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] updated_port = self._update_port( [ 773.899134] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 773.899134] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] _ensure_no_port_binding_failure(port) [ 773.899134] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 773.899134] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] raise exception.PortBindingFailed(port_id=port['id']) [ 773.899134] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] nova.exception.PortBindingFailed: Binding failed for port ede41595-dbc3-4717-a3d6-e73535f68a35, please check neutron logs for more information. [ 773.899134] env[63593]: ERROR nova.compute.manager [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] [ 773.899134] env[63593]: DEBUG nova.compute.utils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Binding failed for port ede41595-dbc3-4717-a3d6-e73535f68a35, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 773.899955] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.420s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 773.903811] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Build of instance 006d2ab8-e38b-4b69-b3b2-115c39e44218 was re-scheduled: Binding failed for port ede41595-dbc3-4717-a3d6-e73535f68a35, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 773.906060] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 773.906291] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquiring lock "refresh_cache-006d2ab8-e38b-4b69-b3b2-115c39e44218" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 773.906439] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Acquired lock "refresh_cache-006d2ab8-e38b-4b69-b3b2-115c39e44218" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 773.906596] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 774.129551] env[63593]: INFO nova.compute.manager [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 70595430-6ab6-49d4-b8fe-131bd44de838] Took 1.03 seconds to deallocate network for instance. [ 774.221066] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 774.435301] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 774.743414] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 774.747206] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 774.841061] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44ecba2d-bde9-4116-9597-17def45df54e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.851026] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28fdd279-39ae-4d1f-a5bf-2475f1c558d0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.880926] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1871f0bc-0896-4276-98ba-0e7a8d4cd975 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.888349] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5dac1742-2143-4b48-894c-61c1812ff847 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.901722] env[63593]: DEBUG nova.compute.provider_tree [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 775.159769] env[63593]: INFO nova.scheduler.client.report [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Deleted allocations for instance 70595430-6ab6-49d4-b8fe-131bd44de838 [ 775.250286] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Releasing lock "refresh_cache-006d2ab8-e38b-4b69-b3b2-115c39e44218" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 775.253018] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 775.253018] env[63593]: DEBUG nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 775.253018] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 775.274681] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 775.405019] env[63593]: DEBUG nova.scheduler.client.report [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 775.670327] env[63593]: DEBUG oslo_concurrency.lockutils [None req-dc3fd5b0-ecb1-434c-b606-d2c088b2a145 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "70595430-6ab6-49d4-b8fe-131bd44de838" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 120.727s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 775.777636] env[63593]: DEBUG nova.network.neutron [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 775.910206] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.010s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 775.911306] env[63593]: ERROR nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133, please check neutron logs for more information. [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Traceback (most recent call last): [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self.driver.spawn(context, instance, image_meta, [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] vm_ref = self.build_virtual_machine(instance, [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] vif_infos = vmwarevif.get_vif_info(self._session, [ 775.911306] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] for vif in network_info: [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] return self._sync_wrapper(fn, *args, **kwargs) [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self.wait() [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self[:] = self._gt.wait() [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] return self._exit_event.wait() [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] current.throw(*self._exc) [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 775.911646] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] result = function(*args, **kwargs) [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] return func(*args, **kwargs) [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] raise e [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] nwinfo = self.network_api.allocate_for_instance( [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] created_port_ids = self._update_ports_for_instance( [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] with excutils.save_and_reraise_exception(): [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] self.force_reraise() [ 775.911968] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 775.912351] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] raise self.value [ 775.912351] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 775.912351] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] updated_port = self._update_port( [ 775.912351] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 775.912351] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] _ensure_no_port_binding_failure(port) [ 775.912351] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 775.912351] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] raise exception.PortBindingFailed(port_id=port['id']) [ 775.912351] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] nova.exception.PortBindingFailed: Binding failed for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133, please check neutron logs for more information. [ 775.912351] env[63593]: ERROR nova.compute.manager [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] [ 775.912351] env[63593]: DEBUG nova.compute.utils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Binding failed for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 775.912807] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.250s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 775.915780] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Build of instance eddf892e-9706-4d33-8241-f8df6c29dee4 was re-scheduled: Binding failed for port 94ea33c3-5b0a-4759-b18b-6ce4367ea133, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 775.916260] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 775.916477] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Acquiring lock "refresh_cache-eddf892e-9706-4d33-8241-f8df6c29dee4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 775.916620] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Acquired lock "refresh_cache-eddf892e-9706-4d33-8241-f8df6c29dee4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 775.916773] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 776.173250] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 776.282956] env[63593]: INFO nova.compute.manager [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] [instance: 006d2ab8-e38b-4b69-b3b2-115c39e44218] Took 1.03 seconds to deallocate network for instance. [ 776.456230] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 776.555558] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 776.688599] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 776.719380] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80e37896-75f0-489b-9824-fbbdd0760440 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.728180] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f09d0692-8d88-4c1c-a24c-9b2f31a176cc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.758952] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffdfbc4c-aaea-457b-b687-f2db4166998c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.765843] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9360f78a-5945-49cd-a49a-c986c402a89c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.778915] env[63593]: DEBUG nova.compute.provider_tree [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 777.060907] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Releasing lock "refresh_cache-eddf892e-9706-4d33-8241-f8df6c29dee4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 777.061184] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 777.061372] env[63593]: DEBUG nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 777.061540] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 777.079102] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 777.282497] env[63593]: DEBUG nova.scheduler.client.report [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 777.330167] env[63593]: INFO nova.scheduler.client.report [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Deleted allocations for instance 006d2ab8-e38b-4b69-b3b2-115c39e44218 [ 777.581969] env[63593]: DEBUG nova.network.neutron [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 777.789618] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.877s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.790567] env[63593]: ERROR nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port abd62477-64c2-490d-ba04-a0168d169e67, please check neutron logs for more information. [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Traceback (most recent call last): [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self.driver.spawn(context, instance, image_meta, [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self._vmops.spawn(context, instance, image_meta, injected_files, [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] vm_ref = self.build_virtual_machine(instance, [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] vif_infos = vmwarevif.get_vif_info(self._session, [ 777.790567] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] for vif in network_info: [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] return self._sync_wrapper(fn, *args, **kwargs) [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self.wait() [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self[:] = self._gt.wait() [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] return self._exit_event.wait() [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] current.throw(*self._exc) [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 777.790877] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] result = function(*args, **kwargs) [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] return func(*args, **kwargs) [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] raise e [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] nwinfo = self.network_api.allocate_for_instance( [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] created_port_ids = self._update_ports_for_instance( [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] with excutils.save_and_reraise_exception(): [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] self.force_reraise() [ 777.791202] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 777.791570] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] raise self.value [ 777.791570] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 777.791570] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] updated_port = self._update_port( [ 777.791570] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 777.791570] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] _ensure_no_port_binding_failure(port) [ 777.791570] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 777.791570] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] raise exception.PortBindingFailed(port_id=port['id']) [ 777.791570] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] nova.exception.PortBindingFailed: Binding failed for port abd62477-64c2-490d-ba04-a0168d169e67, please check neutron logs for more information. [ 777.791570] env[63593]: ERROR nova.compute.manager [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] [ 777.791570] env[63593]: DEBUG nova.compute.utils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Binding failed for port abd62477-64c2-490d-ba04-a0168d169e67, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 777.793050] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.406s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 777.795350] env[63593]: INFO nova.compute.claims [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 777.798677] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Build of instance c99cbfae-db59-4b84-b8ab-09b84ebe4623 was re-scheduled: Binding failed for port abd62477-64c2-490d-ba04-a0168d169e67, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 777.801105] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 777.801653] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Acquiring lock "refresh_cache-c99cbfae-db59-4b84-b8ab-09b84ebe4623" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 777.801868] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Acquired lock "refresh_cache-c99cbfae-db59-4b84-b8ab-09b84ebe4623" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 777.802456] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 777.839901] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e90023-2436-4478-a151-1cfa993cc668 tempest-ServerRescueNegativeTestJSON-772992035 tempest-ServerRescueNegativeTestJSON-772992035-project-member] Lock "006d2ab8-e38b-4b69-b3b2-115c39e44218" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 120.380s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 778.086068] env[63593]: INFO nova.compute.manager [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] [instance: eddf892e-9706-4d33-8241-f8df6c29dee4] Took 1.02 seconds to deallocate network for instance. [ 778.327057] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 778.349030] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 778.463470] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 778.867732] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 778.969667] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Releasing lock "refresh_cache-c99cbfae-db59-4b84-b8ab-09b84ebe4623" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 778.969667] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 778.969667] env[63593]: DEBUG nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 778.969667] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 778.998343] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 779.126139] env[63593]: INFO nova.scheduler.client.report [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Deleted allocations for instance eddf892e-9706-4d33-8241-f8df6c29dee4 [ 779.189798] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51e34788-a63b-486e-9c34-9e1e3252f187 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.197692] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdf06835-fea8-430d-8eb8-a72dcadcb53a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.230432] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e81b693-b9d6-4c20-8ac5-319e0e4acedf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.238356] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f80643d7-bff3-487e-96ce-82f938b595da {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.251986] env[63593]: DEBUG nova.compute.provider_tree [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 779.502993] env[63593]: DEBUG nova.network.neutron [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 779.638545] env[63593]: DEBUG oslo_concurrency.lockutils [None req-862ee91a-0ef4-4c59-91e3-93f3ad0026a4 tempest-InstanceActionsNegativeTestJSON-1962430984 tempest-InstanceActionsNegativeTestJSON-1962430984-project-member] Lock "eddf892e-9706-4d33-8241-f8df6c29dee4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 120.488s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 779.755064] env[63593]: DEBUG nova.scheduler.client.report [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 780.006448] env[63593]: INFO nova.compute.manager [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] [instance: c99cbfae-db59-4b84-b8ab-09b84ebe4623] Took 1.04 seconds to deallocate network for instance. [ 780.141106] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 780.260650] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.467s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 780.261501] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 780.264506] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.558s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.666029] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.773206] env[63593]: DEBUG nova.compute.utils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 780.780221] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 780.780442] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 780.850759] env[63593]: DEBUG nova.policy [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4a89027e1af84390b6b20b85d436b7b9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3a412e2eea2b4b81b4e42378c299ba1d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 781.041509] env[63593]: INFO nova.scheduler.client.report [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Deleted allocations for instance c99cbfae-db59-4b84-b8ab-09b84ebe4623 [ 781.164132] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ad15a44-f83c-48e6-ab19-b13fc870c5ce {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.174847] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7aa2f67f-a18f-4537-a422-20d354ad6e4b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.182786] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Successfully created port: 12f0371d-1a86-48e3-81e7-539adfc84de8 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 781.228669] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a200550-214e-44e6-906d-9731bf077e53 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.236821] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9c230c7-9b26-43fd-ac2f-a9fa20630abe {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 781.249592] env[63593]: DEBUG nova.compute.provider_tree [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 781.279634] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 781.557961] env[63593]: DEBUG oslo_concurrency.lockutils [None req-86806993-73e3-45bd-b234-0f2e0ce21829 tempest-ServerRescueTestJSONUnderV235-2060756734 tempest-ServerRescueTestJSONUnderV235-2060756734-project-member] Lock "c99cbfae-db59-4b84-b8ab-09b84ebe4623" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 121.402s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 781.752895] env[63593]: DEBUG nova.scheduler.client.report [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 782.060656] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 782.258862] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.994s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.259687] env[63593]: ERROR nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6d345b99-0643-41ed-ada2-716e98d37f66, please check neutron logs for more information. [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Traceback (most recent call last): [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self.driver.spawn(context, instance, image_meta, [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] vm_ref = self.build_virtual_machine(instance, [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] vif_infos = vmwarevif.get_vif_info(self._session, [ 782.259687] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] for vif in network_info: [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] return self._sync_wrapper(fn, *args, **kwargs) [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self.wait() [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self[:] = self._gt.wait() [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] return self._exit_event.wait() [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] current.throw(*self._exc) [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 782.260152] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] result = function(*args, **kwargs) [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] return func(*args, **kwargs) [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] raise e [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] nwinfo = self.network_api.allocate_for_instance( [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] created_port_ids = self._update_ports_for_instance( [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] with excutils.save_and_reraise_exception(): [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] self.force_reraise() [ 782.260483] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 782.260814] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] raise self.value [ 782.260814] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 782.260814] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] updated_port = self._update_port( [ 782.260814] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 782.260814] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] _ensure_no_port_binding_failure(port) [ 782.260814] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 782.260814] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] raise exception.PortBindingFailed(port_id=port['id']) [ 782.260814] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] nova.exception.PortBindingFailed: Binding failed for port 6d345b99-0643-41ed-ada2-716e98d37f66, please check neutron logs for more information. [ 782.260814] env[63593]: ERROR nova.compute.manager [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] [ 782.260814] env[63593]: DEBUG nova.compute.utils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Binding failed for port 6d345b99-0643-41ed-ada2-716e98d37f66, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 782.262488] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.381s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.265460] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Build of instance 4f1c0deb-5759-44d2-a2ef-f94e1905949d was re-scheduled: Binding failed for port 6d345b99-0643-41ed-ada2-716e98d37f66, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 782.266503] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 782.266503] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Acquiring lock "refresh_cache-4f1c0deb-5759-44d2-a2ef-f94e1905949d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 782.266503] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Acquired lock "refresh_cache-4f1c0deb-5759-44d2-a2ef-f94e1905949d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 782.266823] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 782.288409] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 782.319236] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 782.320024] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 782.320024] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 782.324020] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 782.324020] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 782.324020] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 782.324020] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 782.324020] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 782.324378] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 782.324378] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 782.324378] env[63593]: DEBUG nova.virt.hardware [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 782.324378] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3ddb9a6-53b5-4f2d-8138-f83d514252eb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 782.335071] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1874d3a1-2a9f-4c46-9eda-607478a2b65b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 782.486837] env[63593]: DEBUG nova.compute.manager [req-1f0ed772-e499-4a75-bd08-c96afd186a42 req-e50962b4-64d8-4d0c-aa17-bbb06b8add0d service nova] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Received event network-changed-12f0371d-1a86-48e3-81e7-539adfc84de8 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 782.487038] env[63593]: DEBUG nova.compute.manager [req-1f0ed772-e499-4a75-bd08-c96afd186a42 req-e50962b4-64d8-4d0c-aa17-bbb06b8add0d service nova] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Refreshing instance network info cache due to event network-changed-12f0371d-1a86-48e3-81e7-539adfc84de8. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 782.487297] env[63593]: DEBUG oslo_concurrency.lockutils [req-1f0ed772-e499-4a75-bd08-c96afd186a42 req-e50962b4-64d8-4d0c-aa17-bbb06b8add0d service nova] Acquiring lock "refresh_cache-b9bd0d9b-322f-476a-ae3f-792ccd869529" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 782.487535] env[63593]: DEBUG oslo_concurrency.lockutils [req-1f0ed772-e499-4a75-bd08-c96afd186a42 req-e50962b4-64d8-4d0c-aa17-bbb06b8add0d service nova] Acquired lock "refresh_cache-b9bd0d9b-322f-476a-ae3f-792ccd869529" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 782.487609] env[63593]: DEBUG nova.network.neutron [req-1f0ed772-e499-4a75-bd08-c96afd186a42 req-e50962b4-64d8-4d0c-aa17-bbb06b8add0d service nova] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Refreshing network info cache for port 12f0371d-1a86-48e3-81e7-539adfc84de8 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 782.597829] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.808855] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 782.849280] env[63593]: ERROR nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 12f0371d-1a86-48e3-81e7-539adfc84de8, please check neutron logs for more information. [ 782.849280] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 782.849280] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 782.849280] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 782.849280] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 782.849280] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 782.849280] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 782.849280] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 782.849280] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 782.849280] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 782.849280] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 782.849280] env[63593]: ERROR nova.compute.manager raise self.value [ 782.849280] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 782.849280] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 782.849280] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 782.849280] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 782.849837] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 782.849837] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 782.849837] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 12f0371d-1a86-48e3-81e7-539adfc84de8, please check neutron logs for more information. [ 782.849837] env[63593]: ERROR nova.compute.manager [ 782.849837] env[63593]: Traceback (most recent call last): [ 782.849837] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 782.849837] env[63593]: listener.cb(fileno) [ 782.849837] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 782.849837] env[63593]: result = function(*args, **kwargs) [ 782.849837] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 782.849837] env[63593]: return func(*args, **kwargs) [ 782.849837] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 782.849837] env[63593]: raise e [ 782.849837] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 782.849837] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 782.849837] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 782.849837] env[63593]: created_port_ids = self._update_ports_for_instance( [ 782.849837] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 782.849837] env[63593]: with excutils.save_and_reraise_exception(): [ 782.849837] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 782.849837] env[63593]: self.force_reraise() [ 782.849837] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 782.849837] env[63593]: raise self.value [ 782.849837] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 782.849837] env[63593]: updated_port = self._update_port( [ 782.849837] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 782.849837] env[63593]: _ensure_no_port_binding_failure(port) [ 782.849837] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 782.849837] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 782.850685] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 12f0371d-1a86-48e3-81e7-539adfc84de8, please check neutron logs for more information. [ 782.850685] env[63593]: Removing descriptor: 19 [ 782.850685] env[63593]: ERROR nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 12f0371d-1a86-48e3-81e7-539adfc84de8, please check neutron logs for more information. [ 782.850685] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Traceback (most recent call last): [ 782.850685] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 782.850685] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] yield resources [ 782.850685] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 782.850685] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self.driver.spawn(context, instance, image_meta, [ 782.850685] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 782.850685] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self._vmops.spawn(context, instance, image_meta, injected_files, [ 782.850685] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 782.850685] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] vm_ref = self.build_virtual_machine(instance, [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] vif_infos = vmwarevif.get_vif_info(self._session, [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] for vif in network_info: [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] return self._sync_wrapper(fn, *args, **kwargs) [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self.wait() [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self[:] = self._gt.wait() [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] return self._exit_event.wait() [ 782.851087] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] result = hub.switch() [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] return self.greenlet.switch() [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] result = function(*args, **kwargs) [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] return func(*args, **kwargs) [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] raise e [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] nwinfo = self.network_api.allocate_for_instance( [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 782.851487] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] created_port_ids = self._update_ports_for_instance( [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] with excutils.save_and_reraise_exception(): [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self.force_reraise() [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] raise self.value [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] updated_port = self._update_port( [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] _ensure_no_port_binding_failure(port) [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 782.851864] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] raise exception.PortBindingFailed(port_id=port['id']) [ 782.852241] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] nova.exception.PortBindingFailed: Binding failed for port 12f0371d-1a86-48e3-81e7-539adfc84de8, please check neutron logs for more information. [ 782.852241] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] [ 782.852241] env[63593]: INFO nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Terminating instance [ 782.855336] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Acquiring lock "refresh_cache-b9bd0d9b-322f-476a-ae3f-792ccd869529" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 783.013217] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 783.015022] env[63593]: DEBUG nova.network.neutron [req-1f0ed772-e499-4a75-bd08-c96afd186a42 req-e50962b4-64d8-4d0c-aa17-bbb06b8add0d service nova] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 783.127104] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0235566e-81d5-40cb-8346-20c7181b5b35 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.135429] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc56e6ee-fa9d-4f4f-aedd-2c3098c360d2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.168649] env[63593]: DEBUG nova.network.neutron [req-1f0ed772-e499-4a75-bd08-c96afd186a42 req-e50962b4-64d8-4d0c-aa17-bbb06b8add0d service nova] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 783.170607] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52c1e873-1ae6-46ba-bf83-7d6e793a822a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.179711] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5896c5bc-a967-41be-a2f8-bbd1017cadec {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.202970] env[63593]: DEBUG nova.compute.provider_tree [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 783.516713] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Releasing lock "refresh_cache-4f1c0deb-5759-44d2-a2ef-f94e1905949d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 783.516988] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 783.517980] env[63593]: DEBUG nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 783.517980] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 783.539355] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 783.677364] env[63593]: DEBUG oslo_concurrency.lockutils [req-1f0ed772-e499-4a75-bd08-c96afd186a42 req-e50962b4-64d8-4d0c-aa17-bbb06b8add0d service nova] Releasing lock "refresh_cache-b9bd0d9b-322f-476a-ae3f-792ccd869529" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 783.678152] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Acquired lock "refresh_cache-b9bd0d9b-322f-476a-ae3f-792ccd869529" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 783.678352] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 783.706793] env[63593]: DEBUG nova.scheduler.client.report [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 784.042844] env[63593]: DEBUG nova.network.neutron [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 784.202759] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 784.211638] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.949s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 784.212431] env[63593]: ERROR nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 13980774-79a8-45fe-a600-24f8ff744044, please check neutron logs for more information. [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Traceback (most recent call last): [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self.driver.spawn(context, instance, image_meta, [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self._vmops.spawn(context, instance, image_meta, injected_files, [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] vm_ref = self.build_virtual_machine(instance, [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] vif_infos = vmwarevif.get_vif_info(self._session, [ 784.212431] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] for vif in network_info: [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] return self._sync_wrapper(fn, *args, **kwargs) [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self.wait() [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self[:] = self._gt.wait() [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] return self._exit_event.wait() [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] current.throw(*self._exc) [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 784.212787] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] result = function(*args, **kwargs) [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] return func(*args, **kwargs) [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] raise e [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] nwinfo = self.network_api.allocate_for_instance( [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] created_port_ids = self._update_ports_for_instance( [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] with excutils.save_and_reraise_exception(): [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] self.force_reraise() [ 784.213282] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 784.213623] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] raise self.value [ 784.213623] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 784.213623] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] updated_port = self._update_port( [ 784.213623] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 784.213623] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] _ensure_no_port_binding_failure(port) [ 784.213623] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 784.213623] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] raise exception.PortBindingFailed(port_id=port['id']) [ 784.213623] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] nova.exception.PortBindingFailed: Binding failed for port 13980774-79a8-45fe-a600-24f8ff744044, please check neutron logs for more information. [ 784.213623] env[63593]: ERROR nova.compute.manager [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] [ 784.213623] env[63593]: DEBUG nova.compute.utils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Binding failed for port 13980774-79a8-45fe-a600-24f8ff744044, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 784.214594] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.166s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 784.220526] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Build of instance 4da891e9-39e0-4739-b1b9-fb438c1111ab was re-scheduled: Binding failed for port 13980774-79a8-45fe-a600-24f8ff744044, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 784.220526] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 784.220526] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "refresh_cache-4da891e9-39e0-4739-b1b9-fb438c1111ab" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 784.220526] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "refresh_cache-4da891e9-39e0-4739-b1b9-fb438c1111ab" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 784.220928] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 784.512248] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 784.546213] env[63593]: INFO nova.compute.manager [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] [instance: 4f1c0deb-5759-44d2-a2ef-f94e1905949d] Took 1.03 seconds to deallocate network for instance. [ 784.652742] env[63593]: DEBUG nova.compute.manager [req-927c4a68-22f8-4f1d-8fb1-f9d588a92c0d req-2d556c18-a3d1-4f96-a0be-ed264302090d service nova] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Received event network-vif-deleted-12f0371d-1a86-48e3-81e7-539adfc84de8 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 784.662161] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "991481e2-557e-4222-adfc-d298fa9314a0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 784.662481] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "991481e2-557e-4222-adfc-d298fa9314a0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 784.743878] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 784.845096] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 785.016411] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Releasing lock "refresh_cache-b9bd0d9b-322f-476a-ae3f-792ccd869529" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 785.017089] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 785.017089] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 785.017308] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-276804de-4b43-456f-976e-2e22ac81f63f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.026252] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cca5fa3-209a-4f1f-9bad-581f3c13b847 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.038750] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db646436-7027-4e3a-b0ca-d6358f04fbb4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.045356] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-714a586c-6537-4876-b90f-5989593874da {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.055049] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b9bd0d9b-322f-476a-ae3f-792ccd869529 could not be found. [ 785.055049] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 785.055049] env[63593]: INFO nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Took 0.04 seconds to destroy the instance on the hypervisor. [ 785.055272] env[63593]: DEBUG oslo.service.loopingcall [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 785.055977] env[63593]: DEBUG nova.compute.manager [-] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 785.056139] env[63593]: DEBUG nova.network.neutron [-] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 785.082563] env[63593]: DEBUG nova.network.neutron [-] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 785.084333] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c48b4e20-8dc7-4839-ab06-c5c558af1a70 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.092357] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0d2a6fa-3104-48e0-b435-84e31c064818 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.105753] env[63593]: DEBUG nova.compute.provider_tree [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 785.350013] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "refresh_cache-4da891e9-39e0-4739-b1b9-fb438c1111ab" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 785.350360] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 785.350476] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 785.350641] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 785.368455] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 785.493430] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Acquiring lock "ef8646c6-7ead-4c9f-8e8f-85c758a35006" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 785.493657] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Lock "ef8646c6-7ead-4c9f-8e8f-85c758a35006" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 785.580738] env[63593]: INFO nova.scheduler.client.report [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Deleted allocations for instance 4f1c0deb-5759-44d2-a2ef-f94e1905949d [ 785.587294] env[63593]: DEBUG nova.network.neutron [-] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 785.608073] env[63593]: DEBUG nova.scheduler.client.report [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 785.872931] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 786.089629] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bc21c454-9242-40dc-9aba-730fdb4a562a tempest-ServerActionsV293TestJSON-109874742 tempest-ServerActionsV293TestJSON-109874742-project-member] Lock "4f1c0deb-5759-44d2-a2ef-f94e1905949d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 117.076s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 786.089912] env[63593]: INFO nova.compute.manager [-] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Took 1.03 seconds to deallocate network for instance. [ 786.092815] env[63593]: DEBUG nova.compute.claims [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 786.093067] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 786.113035] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.898s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 786.113679] env[63593]: ERROR nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d1bc7083-f53f-4bee-8d7e-39e81796668a, please check neutron logs for more information. [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Traceback (most recent call last): [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self.driver.spawn(context, instance, image_meta, [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] vm_ref = self.build_virtual_machine(instance, [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] vif_infos = vmwarevif.get_vif_info(self._session, [ 786.113679] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] for vif in network_info: [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] return self._sync_wrapper(fn, *args, **kwargs) [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self.wait() [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self[:] = self._gt.wait() [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] return self._exit_event.wait() [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] current.throw(*self._exc) [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 786.114018] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] result = function(*args, **kwargs) [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] return func(*args, **kwargs) [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] raise e [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] nwinfo = self.network_api.allocate_for_instance( [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] created_port_ids = self._update_ports_for_instance( [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] with excutils.save_and_reraise_exception(): [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] self.force_reraise() [ 786.114360] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 786.114749] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] raise self.value [ 786.114749] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 786.114749] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] updated_port = self._update_port( [ 786.114749] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 786.114749] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] _ensure_no_port_binding_failure(port) [ 786.114749] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 786.114749] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] raise exception.PortBindingFailed(port_id=port['id']) [ 786.114749] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] nova.exception.PortBindingFailed: Binding failed for port d1bc7083-f53f-4bee-8d7e-39e81796668a, please check neutron logs for more information. [ 786.114749] env[63593]: ERROR nova.compute.manager [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] [ 786.114749] env[63593]: DEBUG nova.compute.utils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Binding failed for port d1bc7083-f53f-4bee-8d7e-39e81796668a, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 786.115695] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.226s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 786.117140] env[63593]: INFO nova.compute.claims [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 786.119945] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Build of instance 788e5910-c189-4d13-a2fa-e5d6e915e95d was re-scheduled: Binding failed for port d1bc7083-f53f-4bee-8d7e-39e81796668a, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 786.120387] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 786.120638] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "refresh_cache-788e5910-c189-4d13-a2fa-e5d6e915e95d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 786.120830] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "refresh_cache-788e5910-c189-4d13-a2fa-e5d6e915e95d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 786.121026] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 786.375180] env[63593]: INFO nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 4da891e9-39e0-4739-b1b9-fb438c1111ab] Took 1.02 seconds to deallocate network for instance. [ 786.594080] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 786.643309] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 786.745461] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 787.121014] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 787.248700] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "refresh_cache-788e5910-c189-4d13-a2fa-e5d6e915e95d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 787.249020] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 787.249223] env[63593]: DEBUG nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 787.249450] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 787.266225] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 787.403565] env[63593]: INFO nova.scheduler.client.report [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Deleted allocations for instance 4da891e9-39e0-4739-b1b9-fb438c1111ab [ 787.452653] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c3c0cc0-795f-43b5-9d50-2540e6aae2c8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.460745] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96ed3099-bdba-4c5a-bb58-d1a3a23091c2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.495000] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9475a8db-ba95-46f8-a833-f23aef5c89ba {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.500550] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3cb3a082-be77-4277-9fdb-c358e5abb0ae {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.514033] env[63593]: DEBUG nova.compute.provider_tree [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 787.770420] env[63593]: DEBUG nova.network.neutron [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 787.914613] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "4da891e9-39e0-4739-b1b9-fb438c1111ab" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 117.082s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 788.019192] env[63593]: DEBUG nova.scheduler.client.report [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 788.275550] env[63593]: INFO nova.compute.manager [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: 788e5910-c189-4d13-a2fa-e5d6e915e95d] Took 1.03 seconds to deallocate network for instance. [ 788.417873] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 788.525671] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.410s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 788.526222] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 788.528956] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.786s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 788.530424] env[63593]: INFO nova.compute.claims [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 788.940108] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 789.038214] env[63593]: DEBUG nova.compute.utils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 789.039584] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 789.039763] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 789.108464] env[63593]: DEBUG nova.policy [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '93f4e26c7d9041c791bc7fef3a6f4f39', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'aa0bfd43220648e09d10476f2d99efdd', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 789.318325] env[63593]: INFO nova.scheduler.client.report [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Deleted allocations for instance 788e5910-c189-4d13-a2fa-e5d6e915e95d [ 789.473963] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Successfully created port: 1bc61261-aadf-46a0-af53-6ec5c2632518 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 789.544766] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 789.826627] env[63593]: DEBUG oslo_concurrency.lockutils [None req-41194734-674b-4370-a85c-b4aef56f1dec tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "788e5910-c189-4d13-a2fa-e5d6e915e95d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 118.925s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 789.838317] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca8498d3-677d-49a6-adc6-edba12e1c626 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.846046] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf2586c6-c86b-4130-93e9-9074031c78c4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.876083] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e44b1e8e-21c5-4264-8dd9-aa24c5ebab2a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.883834] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0696472-1eea-41be-8347-c36f1e8e61bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.903649] env[63593]: DEBUG nova.compute.provider_tree [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 790.313204] env[63593]: DEBUG nova.compute.manager [req-813dfef4-3b69-4775-89f9-73e090d78411 req-79bd622d-c1eb-4f45-8409-881a8b1b3186 service nova] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Received event network-changed-1bc61261-aadf-46a0-af53-6ec5c2632518 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 790.313406] env[63593]: DEBUG nova.compute.manager [req-813dfef4-3b69-4775-89f9-73e090d78411 req-79bd622d-c1eb-4f45-8409-881a8b1b3186 service nova] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Refreshing instance network info cache due to event network-changed-1bc61261-aadf-46a0-af53-6ec5c2632518. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 790.313649] env[63593]: DEBUG oslo_concurrency.lockutils [req-813dfef4-3b69-4775-89f9-73e090d78411 req-79bd622d-c1eb-4f45-8409-881a8b1b3186 service nova] Acquiring lock "refresh_cache-9bc65ce3-6459-4356-8a63-74f55b6f3b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 790.313798] env[63593]: DEBUG oslo_concurrency.lockutils [req-813dfef4-3b69-4775-89f9-73e090d78411 req-79bd622d-c1eb-4f45-8409-881a8b1b3186 service nova] Acquired lock "refresh_cache-9bc65ce3-6459-4356-8a63-74f55b6f3b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 790.313988] env[63593]: DEBUG nova.network.neutron [req-813dfef4-3b69-4775-89f9-73e090d78411 req-79bd622d-c1eb-4f45-8409-881a8b1b3186 service nova] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Refreshing network info cache for port 1bc61261-aadf-46a0-af53-6ec5c2632518 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 790.330075] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 790.407196] env[63593]: DEBUG nova.scheduler.client.report [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 790.467991] env[63593]: ERROR nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1bc61261-aadf-46a0-af53-6ec5c2632518, please check neutron logs for more information. [ 790.467991] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 790.467991] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 790.467991] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 790.467991] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 790.467991] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 790.467991] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 790.467991] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 790.467991] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 790.467991] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 790.467991] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 790.467991] env[63593]: ERROR nova.compute.manager raise self.value [ 790.467991] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 790.467991] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 790.467991] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 790.467991] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 790.469373] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 790.469373] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 790.469373] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1bc61261-aadf-46a0-af53-6ec5c2632518, please check neutron logs for more information. [ 790.469373] env[63593]: ERROR nova.compute.manager [ 790.469373] env[63593]: Traceback (most recent call last): [ 790.469373] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 790.469373] env[63593]: listener.cb(fileno) [ 790.469373] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 790.469373] env[63593]: result = function(*args, **kwargs) [ 790.469373] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 790.469373] env[63593]: return func(*args, **kwargs) [ 790.469373] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 790.469373] env[63593]: raise e [ 790.469373] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 790.469373] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 790.469373] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 790.469373] env[63593]: created_port_ids = self._update_ports_for_instance( [ 790.469373] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 790.469373] env[63593]: with excutils.save_and_reraise_exception(): [ 790.469373] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 790.469373] env[63593]: self.force_reraise() [ 790.469373] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 790.469373] env[63593]: raise self.value [ 790.469373] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 790.469373] env[63593]: updated_port = self._update_port( [ 790.469373] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 790.469373] env[63593]: _ensure_no_port_binding_failure(port) [ 790.469373] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 790.469373] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 790.470326] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 1bc61261-aadf-46a0-af53-6ec5c2632518, please check neutron logs for more information. [ 790.470326] env[63593]: Removing descriptor: 19 [ 790.560825] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 790.590798] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 790.592191] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 790.592335] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 790.592531] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 790.592679] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 790.592824] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 790.593040] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 790.593204] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 790.593430] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 790.594096] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 790.594096] env[63593]: DEBUG nova.virt.hardware [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 790.594922] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ba8d33e-78af-476b-8253-4af311f041b7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.606271] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9409f123-b033-4c3f-9a0d-ceffdf6c90da {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.621761] env[63593]: ERROR nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1bc61261-aadf-46a0-af53-6ec5c2632518, please check neutron logs for more information. [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Traceback (most recent call last): [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] yield resources [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self.driver.spawn(context, instance, image_meta, [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self._vmops.spawn(context, instance, image_meta, injected_files, [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] vm_ref = self.build_virtual_machine(instance, [ 790.621761] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] vif_infos = vmwarevif.get_vif_info(self._session, [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] for vif in network_info: [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] return self._sync_wrapper(fn, *args, **kwargs) [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self.wait() [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self[:] = self._gt.wait() [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] return self._exit_event.wait() [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 790.622254] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] current.throw(*self._exc) [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] result = function(*args, **kwargs) [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] return func(*args, **kwargs) [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] raise e [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] nwinfo = self.network_api.allocate_for_instance( [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] created_port_ids = self._update_ports_for_instance( [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] with excutils.save_and_reraise_exception(): [ 790.624644] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self.force_reraise() [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] raise self.value [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] updated_port = self._update_port( [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] _ensure_no_port_binding_failure(port) [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] raise exception.PortBindingFailed(port_id=port['id']) [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] nova.exception.PortBindingFailed: Binding failed for port 1bc61261-aadf-46a0-af53-6ec5c2632518, please check neutron logs for more information. [ 790.625180] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] [ 790.625180] env[63593]: INFO nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Terminating instance [ 790.625560] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "refresh_cache-9bc65ce3-6459-4356-8a63-74f55b6f3b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 790.839455] env[63593]: DEBUG nova.network.neutron [req-813dfef4-3b69-4775-89f9-73e090d78411 req-79bd622d-c1eb-4f45-8409-881a8b1b3186 service nova] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 790.854620] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 790.913528] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.384s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 790.914083] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 790.917078] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.229s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 790.921093] env[63593]: INFO nova.compute.claims [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 790.933128] env[63593]: DEBUG nova.network.neutron [req-813dfef4-3b69-4775-89f9-73e090d78411 req-79bd622d-c1eb-4f45-8409-881a8b1b3186 service nova] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 791.424465] env[63593]: DEBUG nova.compute.utils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 791.429202] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 791.429383] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 791.438416] env[63593]: DEBUG oslo_concurrency.lockutils [req-813dfef4-3b69-4775-89f9-73e090d78411 req-79bd622d-c1eb-4f45-8409-881a8b1b3186 service nova] Releasing lock "refresh_cache-9bc65ce3-6459-4356-8a63-74f55b6f3b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 791.438416] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquired lock "refresh_cache-9bc65ce3-6459-4356-8a63-74f55b6f3b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 791.438416] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 791.500180] env[63593]: DEBUG nova.policy [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '93f4e26c7d9041c791bc7fef3a6f4f39', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'aa0bfd43220648e09d10476f2d99efdd', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 791.753322] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 791.753595] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 791.788193] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "d3a4902f-1829-427c-9f37-6e83ae7265e3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 791.788340] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "d3a4902f-1829-427c-9f37-6e83ae7265e3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 791.922030] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Successfully created port: 27db0986-3363-4052-bbc7-82e184d28792 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 791.930193] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 791.979281] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 792.234322] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 792.321280] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca5bc5b8-e0d2-4ffc-a6dc-0b56eb1f60cc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.328985] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcdce7ed-6789-4a78-8003-140ecc3bbc9c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.362565] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb06f82a-a239-49b3-b27a-1f6d85f43c1a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.369218] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48fc6df6-9312-4441-ab9b-a8293ca1a3e3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.386599] env[63593]: DEBUG nova.compute.provider_tree [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 792.448060] env[63593]: DEBUG nova.compute.manager [req-46017fb8-9835-4d90-ae20-b25b77031b35 req-fccbcadf-3b08-46da-a579-3a48a18878d2 service nova] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Received event network-vif-deleted-1bc61261-aadf-46a0-af53-6ec5c2632518 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 792.578034] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquiring lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 792.578214] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 792.739261] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Releasing lock "refresh_cache-9bc65ce3-6459-4356-8a63-74f55b6f3b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 792.739261] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 792.739261] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 792.739261] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-afbed090-2427-4e62-8a00-e2358a7ead67 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.750051] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13c25a5b-784a-404d-9dfd-87c8e8bf6242 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.774242] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9bc65ce3-6459-4356-8a63-74f55b6f3b47 could not be found. [ 792.774532] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 792.774648] env[63593]: INFO nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Took 0.04 seconds to destroy the instance on the hypervisor. [ 792.774881] env[63593]: DEBUG oslo.service.loopingcall [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 792.775096] env[63593]: DEBUG nova.compute.manager [-] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 792.775194] env[63593]: DEBUG nova.network.neutron [-] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 792.791819] env[63593]: DEBUG nova.network.neutron [-] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 792.890283] env[63593]: DEBUG nova.scheduler.client.report [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 792.949965] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 792.975573] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 792.975860] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 792.976034] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 792.976214] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 792.976367] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 792.976696] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 792.976982] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 792.977166] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 792.977376] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 792.977538] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 792.977716] env[63593]: DEBUG nova.virt.hardware [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 792.978581] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92618a5e-9051-466d-a915-9dd8fe35018b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.988865] env[63593]: ERROR nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 27db0986-3363-4052-bbc7-82e184d28792, please check neutron logs for more information. [ 792.988865] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 792.988865] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 792.988865] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 792.988865] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 792.988865] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 792.988865] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 792.988865] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 792.988865] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 792.988865] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 792.988865] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 792.988865] env[63593]: ERROR nova.compute.manager raise self.value [ 792.988865] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 792.988865] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 792.988865] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 792.988865] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 792.989377] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 792.989377] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 792.989377] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 27db0986-3363-4052-bbc7-82e184d28792, please check neutron logs for more information. [ 792.989377] env[63593]: ERROR nova.compute.manager [ 792.989377] env[63593]: Traceback (most recent call last): [ 792.989377] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 792.989377] env[63593]: listener.cb(fileno) [ 792.989377] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 792.989377] env[63593]: result = function(*args, **kwargs) [ 792.989377] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 792.989377] env[63593]: return func(*args, **kwargs) [ 792.989377] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 792.989377] env[63593]: raise e [ 792.989377] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 792.989377] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 792.989377] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 792.989377] env[63593]: created_port_ids = self._update_ports_for_instance( [ 792.989377] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 792.989377] env[63593]: with excutils.save_and_reraise_exception(): [ 792.989377] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 792.989377] env[63593]: self.force_reraise() [ 792.989377] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 792.989377] env[63593]: raise self.value [ 792.989377] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 792.989377] env[63593]: updated_port = self._update_port( [ 792.989377] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 792.989377] env[63593]: _ensure_no_port_binding_failure(port) [ 792.989377] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 792.989377] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 792.990235] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 27db0986-3363-4052-bbc7-82e184d28792, please check neutron logs for more information. [ 792.990235] env[63593]: Removing descriptor: 19 [ 792.992383] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d838dc0-b2df-4138-8483-ad551c12d747 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 793.010126] env[63593]: ERROR nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 27db0986-3363-4052-bbc7-82e184d28792, please check neutron logs for more information. [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] Traceback (most recent call last): [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] yield resources [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self.driver.spawn(context, instance, image_meta, [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] vm_ref = self.build_virtual_machine(instance, [ 793.010126] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] vif_infos = vmwarevif.get_vif_info(self._session, [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] for vif in network_info: [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] return self._sync_wrapper(fn, *args, **kwargs) [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self.wait() [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self[:] = self._gt.wait() [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] return self._exit_event.wait() [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 793.010480] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] current.throw(*self._exc) [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] result = function(*args, **kwargs) [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] return func(*args, **kwargs) [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] raise e [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] nwinfo = self.network_api.allocate_for_instance( [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] created_port_ids = self._update_ports_for_instance( [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] with excutils.save_and_reraise_exception(): [ 793.010849] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self.force_reraise() [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] raise self.value [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] updated_port = self._update_port( [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] _ensure_no_port_binding_failure(port) [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] raise exception.PortBindingFailed(port_id=port['id']) [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] nova.exception.PortBindingFailed: Binding failed for port 27db0986-3363-4052-bbc7-82e184d28792, please check neutron logs for more information. [ 793.011222] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] [ 793.011222] env[63593]: INFO nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Terminating instance [ 793.013312] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "refresh_cache-b62cc771-973a-4d23-87b4-825211494d9c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 793.013470] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquired lock "refresh_cache-b62cc771-973a-4d23-87b4-825211494d9c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 793.013665] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 793.294557] env[63593]: DEBUG nova.network.neutron [-] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 793.399288] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.482s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 793.400436] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 793.405693] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.538s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 793.408085] env[63593]: INFO nova.compute.claims [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 793.536613] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 793.589678] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 793.796928] env[63593]: INFO nova.compute.manager [-] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Took 1.02 seconds to deallocate network for instance. [ 793.799283] env[63593]: DEBUG nova.compute.claims [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 793.799454] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 793.914932] env[63593]: DEBUG nova.compute.utils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 793.918213] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 793.918396] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 793.957971] env[63593]: DEBUG nova.policy [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2a9e98a21f854388b8ed32d3662263bf', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2f95123fa26d4061b5748fdb689e5985', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 794.094021] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Releasing lock "refresh_cache-b62cc771-973a-4d23-87b4-825211494d9c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 794.094506] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 794.094707] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 794.095026] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0fd46a79-3cf5-4dae-9d01-0893b230631b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.108023] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ee3b180-2ab7-4a7b-a20c-c47fa023f0a8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.128377] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b62cc771-973a-4d23-87b4-825211494d9c could not be found. [ 794.128604] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 794.128779] env[63593]: INFO nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Took 0.03 seconds to destroy the instance on the hypervisor. [ 794.129021] env[63593]: DEBUG oslo.service.loopingcall [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 794.129226] env[63593]: DEBUG nova.compute.manager [-] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 794.129319] env[63593]: DEBUG nova.network.neutron [-] [instance: b62cc771-973a-4d23-87b4-825211494d9c] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 794.147148] env[63593]: DEBUG nova.network.neutron [-] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 794.244521] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Successfully created port: eb902f6f-50ca-4b50-9958-abbdc4057646 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 794.420062] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 794.538546] env[63593]: DEBUG nova.compute.manager [req-0a4b70e0-c25f-437b-9bd4-61f8ab163f6e req-2eebbfc4-c14f-4ec8-b589-9fbc9eba7367 service nova] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Received event network-changed-27db0986-3363-4052-bbc7-82e184d28792 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 794.538546] env[63593]: DEBUG nova.compute.manager [req-0a4b70e0-c25f-437b-9bd4-61f8ab163f6e req-2eebbfc4-c14f-4ec8-b589-9fbc9eba7367 service nova] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Refreshing instance network info cache due to event network-changed-27db0986-3363-4052-bbc7-82e184d28792. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 794.538546] env[63593]: DEBUG oslo_concurrency.lockutils [req-0a4b70e0-c25f-437b-9bd4-61f8ab163f6e req-2eebbfc4-c14f-4ec8-b589-9fbc9eba7367 service nova] Acquiring lock "refresh_cache-b62cc771-973a-4d23-87b4-825211494d9c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 794.538546] env[63593]: DEBUG oslo_concurrency.lockutils [req-0a4b70e0-c25f-437b-9bd4-61f8ab163f6e req-2eebbfc4-c14f-4ec8-b589-9fbc9eba7367 service nova] Acquired lock "refresh_cache-b62cc771-973a-4d23-87b4-825211494d9c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 794.538546] env[63593]: DEBUG nova.network.neutron [req-0a4b70e0-c25f-437b-9bd4-61f8ab163f6e req-2eebbfc4-c14f-4ec8-b589-9fbc9eba7367 service nova] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Refreshing network info cache for port 27db0986-3363-4052-bbc7-82e184d28792 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 794.647413] env[63593]: DEBUG nova.network.neutron [-] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 794.813794] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c895ad92-b39f-4c0f-8dd6-e1f33df0cf43 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.821563] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4aaac309-cf9b-4cd6-bc25-c810b7ef92d9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.850189] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-872c070b-4646-4b77-ad59-bff1df106cd0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.856982] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59f90550-e4b1-4273-aae0-b25056019ffd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.869450] env[63593]: DEBUG nova.compute.provider_tree [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 795.054279] env[63593]: DEBUG nova.network.neutron [req-0a4b70e0-c25f-437b-9bd4-61f8ab163f6e req-2eebbfc4-c14f-4ec8-b589-9fbc9eba7367 service nova] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 795.100748] env[63593]: DEBUG nova.network.neutron [req-0a4b70e0-c25f-437b-9bd4-61f8ab163f6e req-2eebbfc4-c14f-4ec8-b589-9fbc9eba7367 service nova] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 795.149915] env[63593]: INFO nova.compute.manager [-] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Took 1.02 seconds to deallocate network for instance. [ 795.152653] env[63593]: DEBUG nova.compute.claims [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 795.152836] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 795.277176] env[63593]: ERROR nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port eb902f6f-50ca-4b50-9958-abbdc4057646, please check neutron logs for more information. [ 795.277176] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 795.277176] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 795.277176] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 795.277176] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 795.277176] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 795.277176] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 795.277176] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 795.277176] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 795.277176] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 795.277176] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 795.277176] env[63593]: ERROR nova.compute.manager raise self.value [ 795.277176] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 795.277176] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 795.277176] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 795.277176] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 795.277755] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 795.277755] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 795.277755] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port eb902f6f-50ca-4b50-9958-abbdc4057646, please check neutron logs for more information. [ 795.277755] env[63593]: ERROR nova.compute.manager [ 795.277755] env[63593]: Traceback (most recent call last): [ 795.277755] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 795.277755] env[63593]: listener.cb(fileno) [ 795.277755] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 795.277755] env[63593]: result = function(*args, **kwargs) [ 795.277755] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 795.277755] env[63593]: return func(*args, **kwargs) [ 795.277755] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 795.277755] env[63593]: raise e [ 795.277755] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 795.277755] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 795.277755] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 795.277755] env[63593]: created_port_ids = self._update_ports_for_instance( [ 795.277755] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 795.277755] env[63593]: with excutils.save_and_reraise_exception(): [ 795.277755] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 795.277755] env[63593]: self.force_reraise() [ 795.277755] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 795.277755] env[63593]: raise self.value [ 795.277755] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 795.277755] env[63593]: updated_port = self._update_port( [ 795.277755] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 795.277755] env[63593]: _ensure_no_port_binding_failure(port) [ 795.277755] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 795.277755] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 795.278756] env[63593]: nova.exception.PortBindingFailed: Binding failed for port eb902f6f-50ca-4b50-9958-abbdc4057646, please check neutron logs for more information. [ 795.278756] env[63593]: Removing descriptor: 19 [ 795.373027] env[63593]: DEBUG nova.scheduler.client.report [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 795.431216] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 795.456597] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 795.456889] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 795.457066] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 795.457249] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 795.457532] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 795.457692] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 795.457976] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 795.458260] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 795.459055] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 795.459055] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 795.459055] env[63593]: DEBUG nova.virt.hardware [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 795.460219] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d60bc13c-c551-41b8-bf7f-7a187e9c5bfa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.468114] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f9dc357-fe50-478a-a1f9-c4ba92c30a69 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.483880] env[63593]: ERROR nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port eb902f6f-50ca-4b50-9958-abbdc4057646, please check neutron logs for more information. [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] Traceback (most recent call last): [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] yield resources [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self.driver.spawn(context, instance, image_meta, [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self._vmops.spawn(context, instance, image_meta, injected_files, [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] vm_ref = self.build_virtual_machine(instance, [ 795.483880] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] vif_infos = vmwarevif.get_vif_info(self._session, [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] for vif in network_info: [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] return self._sync_wrapper(fn, *args, **kwargs) [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self.wait() [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self[:] = self._gt.wait() [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] return self._exit_event.wait() [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 795.484282] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] current.throw(*self._exc) [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] result = function(*args, **kwargs) [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] return func(*args, **kwargs) [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] raise e [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] nwinfo = self.network_api.allocate_for_instance( [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] created_port_ids = self._update_ports_for_instance( [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] with excutils.save_and_reraise_exception(): [ 795.484717] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self.force_reraise() [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] raise self.value [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] updated_port = self._update_port( [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] _ensure_no_port_binding_failure(port) [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] raise exception.PortBindingFailed(port_id=port['id']) [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] nova.exception.PortBindingFailed: Binding failed for port eb902f6f-50ca-4b50-9958-abbdc4057646, please check neutron logs for more information. [ 795.485136] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] [ 795.485136] env[63593]: INFO nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Terminating instance [ 795.486585] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "refresh_cache-b965361a-6b76-4836-9887-b59ca990fd67" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 795.486745] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquired lock "refresh_cache-b965361a-6b76-4836-9887-b59ca990fd67" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 795.486906] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 795.603564] env[63593]: DEBUG oslo_concurrency.lockutils [req-0a4b70e0-c25f-437b-9bd4-61f8ab163f6e req-2eebbfc4-c14f-4ec8-b589-9fbc9eba7367 service nova] Releasing lock "refresh_cache-b62cc771-973a-4d23-87b4-825211494d9c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 795.603825] env[63593]: DEBUG nova.compute.manager [req-0a4b70e0-c25f-437b-9bd4-61f8ab163f6e req-2eebbfc4-c14f-4ec8-b589-9fbc9eba7367 service nova] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Received event network-vif-deleted-27db0986-3363-4052-bbc7-82e184d28792 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 795.878026] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.472s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 795.878644] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 795.881598] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.217s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 795.882994] env[63593]: INFO nova.compute.claims [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 796.005962] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 796.086218] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 796.388192] env[63593]: DEBUG nova.compute.utils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 796.392439] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 796.392647] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 796.448850] env[63593]: DEBUG nova.policy [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '79470a9d8a6d4e76b7ca84948c4ea7df', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9f0780232f2a46db849695b5490bb673', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 796.575744] env[63593]: DEBUG nova.compute.manager [req-c859b373-4efb-48b9-a1b0-6e607546feac req-591dde1e-ae83-4108-aa87-42a763a53a6d service nova] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Received event network-changed-eb902f6f-50ca-4b50-9958-abbdc4057646 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 796.576008] env[63593]: DEBUG nova.compute.manager [req-c859b373-4efb-48b9-a1b0-6e607546feac req-591dde1e-ae83-4108-aa87-42a763a53a6d service nova] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Refreshing instance network info cache due to event network-changed-eb902f6f-50ca-4b50-9958-abbdc4057646. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 796.576176] env[63593]: DEBUG oslo_concurrency.lockutils [req-c859b373-4efb-48b9-a1b0-6e607546feac req-591dde1e-ae83-4108-aa87-42a763a53a6d service nova] Acquiring lock "refresh_cache-b965361a-6b76-4836-9887-b59ca990fd67" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 796.588783] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Releasing lock "refresh_cache-b965361a-6b76-4836-9887-b59ca990fd67" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 796.591019] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 796.591019] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 796.591019] env[63593]: DEBUG oslo_concurrency.lockutils [req-c859b373-4efb-48b9-a1b0-6e607546feac req-591dde1e-ae83-4108-aa87-42a763a53a6d service nova] Acquired lock "refresh_cache-b965361a-6b76-4836-9887-b59ca990fd67" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 796.591019] env[63593]: DEBUG nova.network.neutron [req-c859b373-4efb-48b9-a1b0-6e607546feac req-591dde1e-ae83-4108-aa87-42a763a53a6d service nova] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Refreshing network info cache for port eb902f6f-50ca-4b50-9958-abbdc4057646 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 796.591019] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8e9c3f54-ad88-4b6e-a820-69b1721d55f7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.601529] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d21d86e4-ece8-4e93-bc34-26585e462837 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.623186] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b965361a-6b76-4836-9887-b59ca990fd67 could not be found. [ 796.623408] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 796.623588] env[63593]: INFO nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Took 0.03 seconds to destroy the instance on the hypervisor. [ 796.623824] env[63593]: DEBUG oslo.service.loopingcall [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 796.624046] env[63593]: DEBUG nova.compute.manager [-] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 796.624144] env[63593]: DEBUG nova.network.neutron [-] [instance: b965361a-6b76-4836-9887-b59ca990fd67] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 796.640974] env[63593]: DEBUG nova.network.neutron [-] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 796.770121] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Successfully created port: 5b08c428-b00e-4a50-856f-c8d3925c051f {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 796.898305] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 797.011922] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Successfully created port: a312596c-886a-43f5-b65f-45d9a212494b {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 797.112945] env[63593]: DEBUG nova.network.neutron [req-c859b373-4efb-48b9-a1b0-6e607546feac req-591dde1e-ae83-4108-aa87-42a763a53a6d service nova] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 797.144058] env[63593]: DEBUG nova.network.neutron [-] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 797.221965] env[63593]: DEBUG nova.network.neutron [req-c859b373-4efb-48b9-a1b0-6e607546feac req-591dde1e-ae83-4108-aa87-42a763a53a6d service nova] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 797.263544] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71bed530-188f-4b9a-a174-493893325d89 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.268417] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Successfully created port: 94cb97e3-0154-432d-918c-e18ee2fc9b83 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 797.272906] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0987dd8f-c03d-40cf-a51b-adb406998c85 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.313563] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14b59d9d-ff3f-447f-9145-5659b5cbbbff {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.319994] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d524adc-878f-42db-a3a8-716132957e76 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.332740] env[63593]: DEBUG nova.compute.provider_tree [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 797.653822] env[63593]: INFO nova.compute.manager [-] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Took 1.03 seconds to deallocate network for instance. [ 797.654614] env[63593]: DEBUG nova.compute.claims [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 797.655060] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 797.726216] env[63593]: DEBUG oslo_concurrency.lockutils [req-c859b373-4efb-48b9-a1b0-6e607546feac req-591dde1e-ae83-4108-aa87-42a763a53a6d service nova] Releasing lock "refresh_cache-b965361a-6b76-4836-9887-b59ca990fd67" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 797.726525] env[63593]: DEBUG nova.compute.manager [req-c859b373-4efb-48b9-a1b0-6e607546feac req-591dde1e-ae83-4108-aa87-42a763a53a6d service nova] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Received event network-vif-deleted-eb902f6f-50ca-4b50-9958-abbdc4057646 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 797.835972] env[63593]: DEBUG nova.scheduler.client.report [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 797.909845] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 797.935932] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 797.936184] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 797.936433] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 797.936507] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 797.936702] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 797.937748] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 797.937748] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 797.937748] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 797.937748] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 797.937748] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 797.938028] env[63593]: DEBUG nova.virt.hardware [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 797.939542] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d5c47cf-96c6-4447-94b8-39424a5dc404 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 797.947998] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0f930c4-47cc-45cb-9f1b-5e0a8c433111 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.178325] env[63593]: ERROR nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 5b08c428-b00e-4a50-856f-c8d3925c051f, please check neutron logs for more information. [ 798.178325] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 798.178325] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 798.178325] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 798.178325] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 798.178325] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 798.178325] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 798.178325] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 798.178325] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 798.178325] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 798.178325] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 798.178325] env[63593]: ERROR nova.compute.manager raise self.value [ 798.178325] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 798.178325] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 798.178325] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 798.178325] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 798.178880] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 798.178880] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 798.178880] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 5b08c428-b00e-4a50-856f-c8d3925c051f, please check neutron logs for more information. [ 798.178880] env[63593]: ERROR nova.compute.manager [ 798.178880] env[63593]: Traceback (most recent call last): [ 798.178880] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 798.178880] env[63593]: listener.cb(fileno) [ 798.178880] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 798.178880] env[63593]: result = function(*args, **kwargs) [ 798.178880] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 798.178880] env[63593]: return func(*args, **kwargs) [ 798.178880] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 798.178880] env[63593]: raise e [ 798.178880] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 798.178880] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 798.178880] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 798.178880] env[63593]: created_port_ids = self._update_ports_for_instance( [ 798.178880] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 798.178880] env[63593]: with excutils.save_and_reraise_exception(): [ 798.178880] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 798.178880] env[63593]: self.force_reraise() [ 798.178880] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 798.178880] env[63593]: raise self.value [ 798.178880] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 798.178880] env[63593]: updated_port = self._update_port( [ 798.178880] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 798.178880] env[63593]: _ensure_no_port_binding_failure(port) [ 798.178880] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 798.178880] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 798.179793] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 5b08c428-b00e-4a50-856f-c8d3925c051f, please check neutron logs for more information. [ 798.179793] env[63593]: Removing descriptor: 19 [ 798.179793] env[63593]: ERROR nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 5b08c428-b00e-4a50-856f-c8d3925c051f, please check neutron logs for more information. [ 798.179793] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Traceback (most recent call last): [ 798.179793] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 798.179793] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] yield resources [ 798.179793] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 798.179793] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self.driver.spawn(context, instance, image_meta, [ 798.179793] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 798.179793] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self._vmops.spawn(context, instance, image_meta, injected_files, [ 798.179793] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 798.179793] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] vm_ref = self.build_virtual_machine(instance, [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] vif_infos = vmwarevif.get_vif_info(self._session, [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] for vif in network_info: [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] return self._sync_wrapper(fn, *args, **kwargs) [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self.wait() [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self[:] = self._gt.wait() [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] return self._exit_event.wait() [ 798.180194] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] result = hub.switch() [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] return self.greenlet.switch() [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] result = function(*args, **kwargs) [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] return func(*args, **kwargs) [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] raise e [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] nwinfo = self.network_api.allocate_for_instance( [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 798.180607] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] created_port_ids = self._update_ports_for_instance( [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] with excutils.save_and_reraise_exception(): [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self.force_reraise() [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] raise self.value [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] updated_port = self._update_port( [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] _ensure_no_port_binding_failure(port) [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 798.181052] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] raise exception.PortBindingFailed(port_id=port['id']) [ 798.181424] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] nova.exception.PortBindingFailed: Binding failed for port 5b08c428-b00e-4a50-856f-c8d3925c051f, please check neutron logs for more information. [ 798.181424] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] [ 798.181424] env[63593]: INFO nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Terminating instance [ 798.182217] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "refresh_cache-077acb40-d01b-42ad-aa68-cbffc4087100" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 798.182368] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquired lock "refresh_cache-077acb40-d01b-42ad-aa68-cbffc4087100" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 798.182522] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 798.340797] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.459s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.341381] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 798.344132] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.746s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 798.345780] env[63593]: INFO nova.compute.claims [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 798.608724] env[63593]: DEBUG nova.compute.manager [req-7087fa6a-6294-4a61-8f79-4774e2225a38 req-38f7889c-e58a-440a-8501-317fa63c4b9d service nova] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Received event network-changed-5b08c428-b00e-4a50-856f-c8d3925c051f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 798.608933] env[63593]: DEBUG nova.compute.manager [req-7087fa6a-6294-4a61-8f79-4774e2225a38 req-38f7889c-e58a-440a-8501-317fa63c4b9d service nova] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Refreshing instance network info cache due to event network-changed-5b08c428-b00e-4a50-856f-c8d3925c051f. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 798.609122] env[63593]: DEBUG oslo_concurrency.lockutils [req-7087fa6a-6294-4a61-8f79-4774e2225a38 req-38f7889c-e58a-440a-8501-317fa63c4b9d service nova] Acquiring lock "refresh_cache-077acb40-d01b-42ad-aa68-cbffc4087100" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 798.716668] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 798.825024] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 798.850608] env[63593]: DEBUG nova.compute.utils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 798.853983] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 798.854166] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 798.903910] env[63593]: DEBUG nova.policy [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd014d8e414164f7ab9fafae7e2d6f73e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '14d96cd638ed4c2abab7fb07b487acd4', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 799.158206] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Successfully created port: d4d842d7-7eaa-4dbc-8da8-96e951e618c6 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 799.327885] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Releasing lock "refresh_cache-077acb40-d01b-42ad-aa68-cbffc4087100" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 799.328343] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 799.328559] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 799.328872] env[63593]: DEBUG oslo_concurrency.lockutils [req-7087fa6a-6294-4a61-8f79-4774e2225a38 req-38f7889c-e58a-440a-8501-317fa63c4b9d service nova] Acquired lock "refresh_cache-077acb40-d01b-42ad-aa68-cbffc4087100" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 799.329052] env[63593]: DEBUG nova.network.neutron [req-7087fa6a-6294-4a61-8f79-4774e2225a38 req-38f7889c-e58a-440a-8501-317fa63c4b9d service nova] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Refreshing network info cache for port 5b08c428-b00e-4a50-856f-c8d3925c051f {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 799.330241] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4e8b8cf5-4bf8-48f7-a79c-56f840cadf71 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.339546] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10019911-650f-4050-a1ef-058cac10b792 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.355186] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 799.366405] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 077acb40-d01b-42ad-aa68-cbffc4087100 could not be found. [ 799.366405] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 799.366405] env[63593]: INFO nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Took 0.04 seconds to destroy the instance on the hypervisor. [ 799.366405] env[63593]: DEBUG oslo.service.loopingcall [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 799.366625] env[63593]: DEBUG nova.compute.manager [-] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 799.366625] env[63593]: DEBUG nova.network.neutron [-] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 799.414397] env[63593]: DEBUG nova.network.neutron [-] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 799.659705] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a74b879-10c7-4b57-90ee-50201d8b8662 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.668911] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e819fe4-2752-4101-bf77-bcfea59c7111 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.700151] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84aa3654-3b5f-4044-b218-618f7fc1145d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.707238] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a5c1e78-0dcf-416e-96a7-129cfab9cae4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.720226] env[63593]: DEBUG nova.compute.provider_tree [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 799.852400] env[63593]: DEBUG nova.network.neutron [req-7087fa6a-6294-4a61-8f79-4774e2225a38 req-38f7889c-e58a-440a-8501-317fa63c4b9d service nova] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 799.980155] env[63593]: DEBUG nova.network.neutron [req-7087fa6a-6294-4a61-8f79-4774e2225a38 req-38f7889c-e58a-440a-8501-317fa63c4b9d service nova] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 800.125953] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 800.127207] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 800.141681] env[63593]: ERROR nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6, please check neutron logs for more information. [ 800.141681] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 800.141681] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 800.141681] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 800.141681] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 800.141681] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 800.141681] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 800.141681] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 800.141681] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 800.141681] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 800.141681] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 800.141681] env[63593]: ERROR nova.compute.manager raise self.value [ 800.141681] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 800.141681] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 800.141681] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 800.141681] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 800.142533] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 800.142533] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 800.142533] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6, please check neutron logs for more information. [ 800.142533] env[63593]: ERROR nova.compute.manager [ 800.142533] env[63593]: Traceback (most recent call last): [ 800.142533] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 800.142533] env[63593]: listener.cb(fileno) [ 800.142533] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 800.142533] env[63593]: result = function(*args, **kwargs) [ 800.142533] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 800.142533] env[63593]: return func(*args, **kwargs) [ 800.142533] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 800.142533] env[63593]: raise e [ 800.142533] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 800.142533] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 800.142533] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 800.142533] env[63593]: created_port_ids = self._update_ports_for_instance( [ 800.142533] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 800.142533] env[63593]: with excutils.save_and_reraise_exception(): [ 800.142533] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 800.142533] env[63593]: self.force_reraise() [ 800.142533] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 800.142533] env[63593]: raise self.value [ 800.142533] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 800.142533] env[63593]: updated_port = self._update_port( [ 800.142533] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 800.142533] env[63593]: _ensure_no_port_binding_failure(port) [ 800.142533] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 800.142533] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 800.143633] env[63593]: nova.exception.PortBindingFailed: Binding failed for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6, please check neutron logs for more information. [ 800.143633] env[63593]: Removing descriptor: 19 [ 800.223377] env[63593]: DEBUG nova.scheduler.client.report [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 800.345292] env[63593]: DEBUG nova.network.neutron [-] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 800.370675] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 800.395571] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 800.395809] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 800.395961] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 800.396147] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 800.396288] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 800.396431] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 800.396633] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 800.396786] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 800.396945] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 800.397114] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 800.397381] env[63593]: DEBUG nova.virt.hardware [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 800.398282] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c909d197-5e7b-4efd-8658-8a623d5fc482 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.406121] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e20ae85e-b1d8-42e1-9c60-6a7f3d18e113 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.420358] env[63593]: ERROR nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6, please check neutron logs for more information. [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Traceback (most recent call last): [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] yield resources [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self.driver.spawn(context, instance, image_meta, [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] vm_ref = self.build_virtual_machine(instance, [ 800.420358] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] vif_infos = vmwarevif.get_vif_info(self._session, [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] for vif in network_info: [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] return self._sync_wrapper(fn, *args, **kwargs) [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self.wait() [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self[:] = self._gt.wait() [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] return self._exit_event.wait() [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 800.420750] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] current.throw(*self._exc) [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] result = function(*args, **kwargs) [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] return func(*args, **kwargs) [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] raise e [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] nwinfo = self.network_api.allocate_for_instance( [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] created_port_ids = self._update_ports_for_instance( [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] with excutils.save_and_reraise_exception(): [ 800.421132] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self.force_reraise() [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] raise self.value [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] updated_port = self._update_port( [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] _ensure_no_port_binding_failure(port) [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] raise exception.PortBindingFailed(port_id=port['id']) [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] nova.exception.PortBindingFailed: Binding failed for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6, please check neutron logs for more information. [ 800.421629] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] [ 800.421629] env[63593]: INFO nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Terminating instance [ 800.422520] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "refresh_cache-1db3e15b-3168-4d01-930f-fe8008b95df6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 800.422680] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquired lock "refresh_cache-1db3e15b-3168-4d01-930f-fe8008b95df6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 800.422841] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 800.482547] env[63593]: DEBUG oslo_concurrency.lockutils [req-7087fa6a-6294-4a61-8f79-4774e2225a38 req-38f7889c-e58a-440a-8501-317fa63c4b9d service nova] Releasing lock "refresh_cache-077acb40-d01b-42ad-aa68-cbffc4087100" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 800.483731] env[63593]: DEBUG nova.compute.manager [req-7087fa6a-6294-4a61-8f79-4774e2225a38 req-38f7889c-e58a-440a-8501-317fa63c4b9d service nova] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Received event network-vif-deleted-5b08c428-b00e-4a50-856f-c8d3925c051f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 800.633935] env[63593]: DEBUG nova.compute.manager [req-970bb6ff-fd0f-4a42-8e2d-b4c262b77ec5 req-bbdfe01d-3c8e-4816-bcd3-29e99525a70f service nova] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Received event network-changed-d4d842d7-7eaa-4dbc-8da8-96e951e618c6 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 800.634037] env[63593]: DEBUG nova.compute.manager [req-970bb6ff-fd0f-4a42-8e2d-b4c262b77ec5 req-bbdfe01d-3c8e-4816-bcd3-29e99525a70f service nova] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Refreshing instance network info cache due to event network-changed-d4d842d7-7eaa-4dbc-8da8-96e951e618c6. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 800.634217] env[63593]: DEBUG oslo_concurrency.lockutils [req-970bb6ff-fd0f-4a42-8e2d-b4c262b77ec5 req-bbdfe01d-3c8e-4816-bcd3-29e99525a70f service nova] Acquiring lock "refresh_cache-1db3e15b-3168-4d01-930f-fe8008b95df6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 800.634361] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 800.634665] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 800.634665] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 800.728032] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.383s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.728905] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 800.731390] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.638s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.847820] env[63593]: INFO nova.compute.manager [-] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Took 1.48 seconds to deallocate network for instance. [ 800.850084] env[63593]: DEBUG nova.compute.claims [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 800.850269] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.949435] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 801.035172] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 801.139886] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 801.140061] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 801.140196] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 801.140321] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 801.140443] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 801.140561] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 801.140677] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 801.140795] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 801.140984] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 801.142246] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 801.142422] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 801.142574] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 801.142718] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 801.142860] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 801.142984] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 801.143143] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 801.233345] env[63593]: DEBUG nova.compute.utils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 801.234718] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 801.234877] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 801.279024] env[63593]: DEBUG nova.policy [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ce148acb1a3343e9abd803c1d915d3d3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'bf36141217ad4f1da316aec34919a2fc', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 801.533832] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5112ccc2-4537-4fe4-b80e-54f442865d44 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.537715] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Releasing lock "refresh_cache-1db3e15b-3168-4d01-930f-fe8008b95df6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 801.537715] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 801.537715] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 801.538263] env[63593]: DEBUG oslo_concurrency.lockutils [req-970bb6ff-fd0f-4a42-8e2d-b4c262b77ec5 req-bbdfe01d-3c8e-4816-bcd3-29e99525a70f service nova] Acquired lock "refresh_cache-1db3e15b-3168-4d01-930f-fe8008b95df6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 801.538396] env[63593]: DEBUG nova.network.neutron [req-970bb6ff-fd0f-4a42-8e2d-b4c262b77ec5 req-bbdfe01d-3c8e-4816-bcd3-29e99525a70f service nova] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Refreshing network info cache for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 801.539379] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-fc4a0184-de5b-48de-93e8-ca467c401d60 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.544127] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9362eb1-288f-450a-ad5e-441bfa0245bf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.552871] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a49a9f2c-f6e4-4e62-91b7-2b65174a13ad {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.587849] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Successfully created port: 399af546-2adb-454f-b248-8377b08a6d41 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 801.590621] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-863ce20c-c25a-40e4-9569-d44755731153 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.598101] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-542a8fb4-d8d7-4ff8-af73-d8dae3cc5983 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.605542] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1db3e15b-3168-4d01-930f-fe8008b95df6 could not be found. [ 801.605800] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 801.606033] env[63593]: INFO nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Took 0.07 seconds to destroy the instance on the hypervisor. [ 801.606315] env[63593]: DEBUG oslo.service.loopingcall [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 801.606861] env[63593]: DEBUG nova.compute.manager [-] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 801.606999] env[63593]: DEBUG nova.network.neutron [-] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 801.616219] env[63593]: DEBUG nova.compute.provider_tree [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 801.636794] env[63593]: DEBUG nova.network.neutron [-] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 801.646311] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 801.738142] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 802.067732] env[63593]: DEBUG nova.network.neutron [req-970bb6ff-fd0f-4a42-8e2d-b4c262b77ec5 req-bbdfe01d-3c8e-4816-bcd3-29e99525a70f service nova] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 802.119419] env[63593]: DEBUG nova.scheduler.client.report [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 802.139112] env[63593]: DEBUG nova.network.neutron [-] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 802.170762] env[63593]: DEBUG nova.network.neutron [req-970bb6ff-fd0f-4a42-8e2d-b4c262b77ec5 req-bbdfe01d-3c8e-4816-bcd3-29e99525a70f service nova] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 802.497790] env[63593]: ERROR nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 399af546-2adb-454f-b248-8377b08a6d41, please check neutron logs for more information. [ 802.497790] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 802.497790] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 802.497790] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 802.497790] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 802.497790] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 802.497790] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 802.497790] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 802.497790] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 802.497790] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 802.497790] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 802.497790] env[63593]: ERROR nova.compute.manager raise self.value [ 802.497790] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 802.497790] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 802.497790] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 802.497790] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 802.498458] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 802.498458] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 802.498458] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 399af546-2adb-454f-b248-8377b08a6d41, please check neutron logs for more information. [ 802.498458] env[63593]: ERROR nova.compute.manager [ 802.498458] env[63593]: Traceback (most recent call last): [ 802.498458] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 802.498458] env[63593]: listener.cb(fileno) [ 802.498458] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 802.498458] env[63593]: result = function(*args, **kwargs) [ 802.498458] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 802.498458] env[63593]: return func(*args, **kwargs) [ 802.498458] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 802.498458] env[63593]: raise e [ 802.498458] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 802.498458] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 802.498458] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 802.498458] env[63593]: created_port_ids = self._update_ports_for_instance( [ 802.498458] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 802.498458] env[63593]: with excutils.save_and_reraise_exception(): [ 802.498458] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 802.498458] env[63593]: self.force_reraise() [ 802.498458] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 802.498458] env[63593]: raise self.value [ 802.498458] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 802.498458] env[63593]: updated_port = self._update_port( [ 802.498458] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 802.498458] env[63593]: _ensure_no_port_binding_failure(port) [ 802.498458] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 802.498458] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 802.499563] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 399af546-2adb-454f-b248-8377b08a6d41, please check neutron logs for more information. [ 802.499563] env[63593]: Removing descriptor: 19 [ 802.624668] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.893s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.625318] env[63593]: ERROR nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 12f0371d-1a86-48e3-81e7-539adfc84de8, please check neutron logs for more information. [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Traceback (most recent call last): [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self.driver.spawn(context, instance, image_meta, [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self._vmops.spawn(context, instance, image_meta, injected_files, [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] vm_ref = self.build_virtual_machine(instance, [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] vif_infos = vmwarevif.get_vif_info(self._session, [ 802.625318] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] for vif in network_info: [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] return self._sync_wrapper(fn, *args, **kwargs) [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self.wait() [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self[:] = self._gt.wait() [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] return self._exit_event.wait() [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] result = hub.switch() [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 802.625745] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] return self.greenlet.switch() [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] result = function(*args, **kwargs) [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] return func(*args, **kwargs) [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] raise e [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] nwinfo = self.network_api.allocate_for_instance( [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] created_port_ids = self._update_ports_for_instance( [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] with excutils.save_and_reraise_exception(): [ 802.626166] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] self.force_reraise() [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] raise self.value [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] updated_port = self._update_port( [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] _ensure_no_port_binding_failure(port) [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] raise exception.PortBindingFailed(port_id=port['id']) [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] nova.exception.PortBindingFailed: Binding failed for port 12f0371d-1a86-48e3-81e7-539adfc84de8, please check neutron logs for more information. [ 802.626562] env[63593]: ERROR nova.compute.manager [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] [ 802.626876] env[63593]: DEBUG nova.compute.utils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Binding failed for port 12f0371d-1a86-48e3-81e7-539adfc84de8, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 802.627286] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.506s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.628763] env[63593]: INFO nova.compute.claims [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 802.631311] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Build of instance b9bd0d9b-322f-476a-ae3f-792ccd869529 was re-scheduled: Binding failed for port 12f0371d-1a86-48e3-81e7-539adfc84de8, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 802.631732] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 802.631950] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Acquiring lock "refresh_cache-b9bd0d9b-322f-476a-ae3f-792ccd869529" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 802.632106] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Acquired lock "refresh_cache-b9bd0d9b-322f-476a-ae3f-792ccd869529" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 802.632260] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 802.641674] env[63593]: INFO nova.compute.manager [-] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Took 1.03 seconds to deallocate network for instance. [ 802.643432] env[63593]: DEBUG nova.compute.claims [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 802.643432] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.657368] env[63593]: DEBUG nova.compute.manager [req-3de003c6-43b3-4eeb-b467-aa6ee3b4187b req-5e4ea7ff-f284-45ef-8ae4-956bfea1802b service nova] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Received event network-changed-399af546-2adb-454f-b248-8377b08a6d41 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 802.657744] env[63593]: DEBUG nova.compute.manager [req-3de003c6-43b3-4eeb-b467-aa6ee3b4187b req-5e4ea7ff-f284-45ef-8ae4-956bfea1802b service nova] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Refreshing instance network info cache due to event network-changed-399af546-2adb-454f-b248-8377b08a6d41. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 802.657834] env[63593]: DEBUG oslo_concurrency.lockutils [req-3de003c6-43b3-4eeb-b467-aa6ee3b4187b req-5e4ea7ff-f284-45ef-8ae4-956bfea1802b service nova] Acquiring lock "refresh_cache-b27b2ed8-ec99-4962-8742-3a8264906b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 802.658019] env[63593]: DEBUG oslo_concurrency.lockutils [req-3de003c6-43b3-4eeb-b467-aa6ee3b4187b req-5e4ea7ff-f284-45ef-8ae4-956bfea1802b service nova] Acquired lock "refresh_cache-b27b2ed8-ec99-4962-8742-3a8264906b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 802.658100] env[63593]: DEBUG nova.network.neutron [req-3de003c6-43b3-4eeb-b467-aa6ee3b4187b req-5e4ea7ff-f284-45ef-8ae4-956bfea1802b service nova] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Refreshing network info cache for port 399af546-2adb-454f-b248-8377b08a6d41 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 802.673141] env[63593]: DEBUG oslo_concurrency.lockutils [req-970bb6ff-fd0f-4a42-8e2d-b4c262b77ec5 req-bbdfe01d-3c8e-4816-bcd3-29e99525a70f service nova] Releasing lock "refresh_cache-1db3e15b-3168-4d01-930f-fe8008b95df6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 802.673354] env[63593]: DEBUG nova.compute.manager [req-970bb6ff-fd0f-4a42-8e2d-b4c262b77ec5 req-bbdfe01d-3c8e-4816-bcd3-29e99525a70f service nova] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Received event network-vif-deleted-d4d842d7-7eaa-4dbc-8da8-96e951e618c6 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 802.747291] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 802.772124] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 802.772376] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 802.772526] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 802.772704] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 802.772846] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 802.772987] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 802.773211] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 802.773365] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 802.773533] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 802.774016] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 802.774977] env[63593]: DEBUG nova.virt.hardware [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 802.775528] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35676d26-0c5d-410f-9998-753ef607e790 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.783597] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f92b91fb-8235-4cea-9d3e-221b1b710437 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.797338] env[63593]: ERROR nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 399af546-2adb-454f-b248-8377b08a6d41, please check neutron logs for more information. [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Traceback (most recent call last): [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] yield resources [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self.driver.spawn(context, instance, image_meta, [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self._vmops.spawn(context, instance, image_meta, injected_files, [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] vm_ref = self.build_virtual_machine(instance, [ 802.797338] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] vif_infos = vmwarevif.get_vif_info(self._session, [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] for vif in network_info: [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] return self._sync_wrapper(fn, *args, **kwargs) [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self.wait() [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self[:] = self._gt.wait() [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] return self._exit_event.wait() [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 802.797867] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] current.throw(*self._exc) [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] result = function(*args, **kwargs) [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] return func(*args, **kwargs) [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] raise e [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] nwinfo = self.network_api.allocate_for_instance( [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] created_port_ids = self._update_ports_for_instance( [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] with excutils.save_and_reraise_exception(): [ 802.798351] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self.force_reraise() [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] raise self.value [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] updated_port = self._update_port( [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] _ensure_no_port_binding_failure(port) [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] raise exception.PortBindingFailed(port_id=port['id']) [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] nova.exception.PortBindingFailed: Binding failed for port 399af546-2adb-454f-b248-8377b08a6d41, please check neutron logs for more information. [ 802.798773] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] [ 802.798773] env[63593]: INFO nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Terminating instance [ 802.799610] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Acquiring lock "refresh_cache-b27b2ed8-ec99-4962-8742-3a8264906b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 803.152857] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 803.177185] env[63593]: DEBUG nova.network.neutron [req-3de003c6-43b3-4eeb-b467-aa6ee3b4187b req-5e4ea7ff-f284-45ef-8ae4-956bfea1802b service nova] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 803.245798] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 803.268734] env[63593]: DEBUG nova.network.neutron [req-3de003c6-43b3-4eeb-b467-aa6ee3b4187b req-5e4ea7ff-f284-45ef-8ae4-956bfea1802b service nova] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 803.748613] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Releasing lock "refresh_cache-b9bd0d9b-322f-476a-ae3f-792ccd869529" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 803.748868] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 803.748944] env[63593]: DEBUG nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 803.749126] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 803.766199] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 803.772377] env[63593]: DEBUG oslo_concurrency.lockutils [req-3de003c6-43b3-4eeb-b467-aa6ee3b4187b req-5e4ea7ff-f284-45ef-8ae4-956bfea1802b service nova] Releasing lock "refresh_cache-b27b2ed8-ec99-4962-8742-3a8264906b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 803.772614] env[63593]: DEBUG nova.compute.manager [req-3de003c6-43b3-4eeb-b467-aa6ee3b4187b req-5e4ea7ff-f284-45ef-8ae4-956bfea1802b service nova] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Received event network-vif-deleted-399af546-2adb-454f-b248-8377b08a6d41 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 803.773399] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Acquired lock "refresh_cache-b27b2ed8-ec99-4962-8742-3a8264906b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 803.773594] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 803.909416] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d53aab4-3414-48d7-90ac-9f0e65a2ae64 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.917281] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83383c28-373a-43e8-9bbb-2b602f431102 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.947700] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be696206-b837-474f-97c8-4e4bc3c06741 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.955273] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04ea8171-5d56-4d35-b8bf-535f617976cd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.969618] env[63593]: DEBUG nova.compute.provider_tree [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 804.269424] env[63593]: DEBUG nova.network.neutron [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 804.292097] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 804.384895] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 804.472305] env[63593]: DEBUG nova.scheduler.client.report [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 804.773876] env[63593]: INFO nova.compute.manager [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] [instance: b9bd0d9b-322f-476a-ae3f-792ccd869529] Took 1.02 seconds to deallocate network for instance. [ 804.887091] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Releasing lock "refresh_cache-b27b2ed8-ec99-4962-8742-3a8264906b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 804.887621] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 804.887822] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 804.888140] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-62befb50-87be-4c94-a90b-6310a0e102f5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 804.897728] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1368a6f-3c39-47f8-b129-a07750167bed {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 804.918561] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b27b2ed8-ec99-4962-8742-3a8264906b47 could not be found. [ 804.918794] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 804.918975] env[63593]: INFO nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Took 0.03 seconds to destroy the instance on the hypervisor. [ 804.919545] env[63593]: DEBUG oslo.service.loopingcall [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 804.919545] env[63593]: DEBUG nova.compute.manager [-] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 804.919545] env[63593]: DEBUG nova.network.neutron [-] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 804.934271] env[63593]: DEBUG nova.network.neutron [-] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 804.978218] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.350s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 804.978218] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 804.980673] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.041s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 804.982996] env[63593]: INFO nova.compute.claims [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 805.436425] env[63593]: DEBUG nova.network.neutron [-] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 805.486235] env[63593]: DEBUG nova.compute.utils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 805.489628] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 805.489628] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 805.537598] env[63593]: DEBUG nova.policy [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd5f23913d4a1411282f081fb5ed46a11', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '220c072768c1441fbfe88f849cd6eb03', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 805.798271] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Successfully created port: c5740fc0-200d-47ec-8497-d1e295793409 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 805.805469] env[63593]: INFO nova.scheduler.client.report [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Deleted allocations for instance b9bd0d9b-322f-476a-ae3f-792ccd869529 [ 805.938866] env[63593]: INFO nova.compute.manager [-] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Took 1.02 seconds to deallocate network for instance. [ 805.941855] env[63593]: DEBUG nova.compute.claims [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 805.941993] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 805.991587] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 806.313641] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cd9557b-e956-4a51-ab35-03ff21d000c8 tempest-ServerRescueTestJSON-718871243 tempest-ServerRescueTestJSON-718871243-project-member] Lock "b9bd0d9b-322f-476a-ae3f-792ccd869529" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 134.459s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 806.317638] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7e1d12b-609b-4fb4-aa65-4f1d8a574fcb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.323967] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2735db3a-b1fe-4bd9-8215-ba8c7830c7bc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.356131] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75204b3a-f321-4409-87a1-0d50d1f51a0f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.363586] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6443dba-0097-4e14-bb5c-5f30d8b48c19 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.376709] env[63593]: DEBUG nova.compute.provider_tree [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 806.628349] env[63593]: DEBUG nova.compute.manager [req-6f2c6484-f770-4db7-b9f3-34fc2704f824 req-b73d1f6a-ea48-4dfe-9727-f90cf5baca6a service nova] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Received event network-changed-c5740fc0-200d-47ec-8497-d1e295793409 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 806.628349] env[63593]: DEBUG nova.compute.manager [req-6f2c6484-f770-4db7-b9f3-34fc2704f824 req-b73d1f6a-ea48-4dfe-9727-f90cf5baca6a service nova] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Refreshing instance network info cache due to event network-changed-c5740fc0-200d-47ec-8497-d1e295793409. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 806.628349] env[63593]: DEBUG oslo_concurrency.lockutils [req-6f2c6484-f770-4db7-b9f3-34fc2704f824 req-b73d1f6a-ea48-4dfe-9727-f90cf5baca6a service nova] Acquiring lock "refresh_cache-a269ba6a-8439-448f-ad51-e3084c89c2a5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 806.628349] env[63593]: DEBUG oslo_concurrency.lockutils [req-6f2c6484-f770-4db7-b9f3-34fc2704f824 req-b73d1f6a-ea48-4dfe-9727-f90cf5baca6a service nova] Acquired lock "refresh_cache-a269ba6a-8439-448f-ad51-e3084c89c2a5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 806.628349] env[63593]: DEBUG nova.network.neutron [req-6f2c6484-f770-4db7-b9f3-34fc2704f824 req-b73d1f6a-ea48-4dfe-9727-f90cf5baca6a service nova] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Refreshing network info cache for port c5740fc0-200d-47ec-8497-d1e295793409 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 806.789949] env[63593]: ERROR nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c5740fc0-200d-47ec-8497-d1e295793409, please check neutron logs for more information. [ 806.789949] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 806.789949] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 806.789949] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 806.789949] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 806.789949] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 806.789949] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 806.789949] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 806.789949] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 806.789949] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 806.789949] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 806.789949] env[63593]: ERROR nova.compute.manager raise self.value [ 806.789949] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 806.789949] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 806.789949] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 806.789949] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 806.790494] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 806.790494] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 806.790494] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c5740fc0-200d-47ec-8497-d1e295793409, please check neutron logs for more information. [ 806.790494] env[63593]: ERROR nova.compute.manager [ 806.790494] env[63593]: Traceback (most recent call last): [ 806.790494] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 806.790494] env[63593]: listener.cb(fileno) [ 806.790494] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 806.790494] env[63593]: result = function(*args, **kwargs) [ 806.790494] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 806.790494] env[63593]: return func(*args, **kwargs) [ 806.790494] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 806.790494] env[63593]: raise e [ 806.790494] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 806.790494] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 806.790494] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 806.790494] env[63593]: created_port_ids = self._update_ports_for_instance( [ 806.790494] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 806.790494] env[63593]: with excutils.save_and_reraise_exception(): [ 806.790494] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 806.790494] env[63593]: self.force_reraise() [ 806.790494] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 806.790494] env[63593]: raise self.value [ 806.790494] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 806.790494] env[63593]: updated_port = self._update_port( [ 806.790494] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 806.790494] env[63593]: _ensure_no_port_binding_failure(port) [ 806.790494] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 806.790494] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 806.791387] env[63593]: nova.exception.PortBindingFailed: Binding failed for port c5740fc0-200d-47ec-8497-d1e295793409, please check neutron logs for more information. [ 806.791387] env[63593]: Removing descriptor: 18 [ 806.820027] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 806.879886] env[63593]: DEBUG nova.scheduler.client.report [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 807.002098] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 807.027310] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 807.027606] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 807.027774] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 807.027955] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 807.028116] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 807.028263] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 807.028464] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 807.028631] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 807.028841] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 807.028990] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 807.029137] env[63593]: DEBUG nova.virt.hardware [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 807.030296] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05af4036-3470-4384-82f7-eaa2f6255663 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.040052] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7737c998-7475-4cf8-ae3d-59db0cd10743 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 807.051772] env[63593]: ERROR nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c5740fc0-200d-47ec-8497-d1e295793409, please check neutron logs for more information. [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Traceback (most recent call last): [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] yield resources [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self.driver.spawn(context, instance, image_meta, [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] vm_ref = self.build_virtual_machine(instance, [ 807.051772] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] vif_infos = vmwarevif.get_vif_info(self._session, [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] for vif in network_info: [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] return self._sync_wrapper(fn, *args, **kwargs) [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self.wait() [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self[:] = self._gt.wait() [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] return self._exit_event.wait() [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 807.052346] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] current.throw(*self._exc) [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] result = function(*args, **kwargs) [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] return func(*args, **kwargs) [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] raise e [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] nwinfo = self.network_api.allocate_for_instance( [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] created_port_ids = self._update_ports_for_instance( [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] with excutils.save_and_reraise_exception(): [ 807.052894] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self.force_reraise() [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] raise self.value [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] updated_port = self._update_port( [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] _ensure_no_port_binding_failure(port) [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] raise exception.PortBindingFailed(port_id=port['id']) [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] nova.exception.PortBindingFailed: Binding failed for port c5740fc0-200d-47ec-8497-d1e295793409, please check neutron logs for more information. [ 807.053511] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] [ 807.053511] env[63593]: INFO nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Terminating instance [ 807.054164] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "refresh_cache-a269ba6a-8439-448f-ad51-e3084c89c2a5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 807.155689] env[63593]: DEBUG nova.network.neutron [req-6f2c6484-f770-4db7-b9f3-34fc2704f824 req-b73d1f6a-ea48-4dfe-9727-f90cf5baca6a service nova] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 807.294102] env[63593]: DEBUG nova.network.neutron [req-6f2c6484-f770-4db7-b9f3-34fc2704f824 req-b73d1f6a-ea48-4dfe-9727-f90cf5baca6a service nova] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 807.343865] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 807.384823] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.404s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 807.385416] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 807.388404] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.534s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 807.389770] env[63593]: INFO nova.compute.claims [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 807.798652] env[63593]: DEBUG oslo_concurrency.lockutils [req-6f2c6484-f770-4db7-b9f3-34fc2704f824 req-b73d1f6a-ea48-4dfe-9727-f90cf5baca6a service nova] Releasing lock "refresh_cache-a269ba6a-8439-448f-ad51-e3084c89c2a5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 807.798652] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "refresh_cache-a269ba6a-8439-448f-ad51-e3084c89c2a5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 807.798652] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 807.894622] env[63593]: DEBUG nova.compute.utils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 807.898114] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 807.898114] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 807.962985] env[63593]: DEBUG nova.policy [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b61df7d41ccd4c0bab6dd620e15de0d8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'eab3efe6479d4b8d92333464a20aab3d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 808.319925] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 808.400936] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 808.428997] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Successfully created port: c825c2c5-b249-4fe4-9844-78dcf55b9a08 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 808.457533] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 808.705549] env[63593]: DEBUG nova.compute.manager [req-823fb7de-16c6-415b-acb8-ae586d8a8383 req-d5888e92-fe79-4b2e-adcb-204199068b43 service nova] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Received event network-vif-deleted-c5740fc0-200d-47ec-8497-d1e295793409 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 808.735102] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a458eb36-78bf-4b5f-9c63-1ef338f8afed {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.743527] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e554edfd-d86e-4597-966b-587d6cd537ee {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.779006] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21cfb3ff-bb03-40e6-83ee-9f247c1c4375 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.786184] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e78cf0e-45df-4a33-af05-65c48be9ef46 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.799877] env[63593]: DEBUG nova.compute.provider_tree [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 808.959709] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "refresh_cache-a269ba6a-8439-448f-ad51-e3084c89c2a5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 808.960186] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 808.960384] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 808.960663] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-675d3552-a42d-4395-b76d-db5525e88a6f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.969734] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d1b426d-b3a2-4c2c-b6af-90c64d677587 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 808.992732] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a269ba6a-8439-448f-ad51-e3084c89c2a5 could not be found. [ 808.992962] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 808.993158] env[63593]: INFO nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Took 0.03 seconds to destroy the instance on the hypervisor. [ 808.993400] env[63593]: DEBUG oslo.service.loopingcall [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 808.993609] env[63593]: DEBUG nova.compute.manager [-] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 808.993705] env[63593]: DEBUG nova.network.neutron [-] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 809.009070] env[63593]: DEBUG nova.network.neutron [-] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 809.303797] env[63593]: DEBUG nova.scheduler.client.report [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 809.418038] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 809.442929] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 809.442929] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 809.442929] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 809.443162] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 809.443162] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 809.443238] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 809.443426] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 809.443599] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 809.443768] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 809.443929] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 809.444109] env[63593]: DEBUG nova.virt.hardware [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 809.444970] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6878e4dd-2248-4edf-8f95-a9c7e36a1dc0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.453070] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f4b3495-b05b-4849-88b4-7667af58f384 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.468412] env[63593]: ERROR nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c825c2c5-b249-4fe4-9844-78dcf55b9a08, please check neutron logs for more information. [ 809.468412] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 809.468412] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 809.468412] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 809.468412] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 809.468412] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 809.468412] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 809.468412] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 809.468412] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 809.468412] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 809.468412] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 809.468412] env[63593]: ERROR nova.compute.manager raise self.value [ 809.468412] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 809.468412] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 809.468412] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 809.468412] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 809.468989] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 809.468989] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 809.468989] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c825c2c5-b249-4fe4-9844-78dcf55b9a08, please check neutron logs for more information. [ 809.468989] env[63593]: ERROR nova.compute.manager [ 809.468989] env[63593]: Traceback (most recent call last): [ 809.468989] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 809.468989] env[63593]: listener.cb(fileno) [ 809.468989] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 809.468989] env[63593]: result = function(*args, **kwargs) [ 809.468989] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 809.468989] env[63593]: return func(*args, **kwargs) [ 809.468989] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 809.468989] env[63593]: raise e [ 809.468989] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 809.468989] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 809.468989] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 809.468989] env[63593]: created_port_ids = self._update_ports_for_instance( [ 809.468989] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 809.468989] env[63593]: with excutils.save_and_reraise_exception(): [ 809.468989] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 809.468989] env[63593]: self.force_reraise() [ 809.468989] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 809.468989] env[63593]: raise self.value [ 809.468989] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 809.468989] env[63593]: updated_port = self._update_port( [ 809.468989] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 809.468989] env[63593]: _ensure_no_port_binding_failure(port) [ 809.468989] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 809.468989] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 809.470030] env[63593]: nova.exception.PortBindingFailed: Binding failed for port c825c2c5-b249-4fe4-9844-78dcf55b9a08, please check neutron logs for more information. [ 809.470030] env[63593]: Removing descriptor: 18 [ 809.470030] env[63593]: ERROR nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c825c2c5-b249-4fe4-9844-78dcf55b9a08, please check neutron logs for more information. [ 809.470030] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Traceback (most recent call last): [ 809.470030] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 809.470030] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] yield resources [ 809.470030] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 809.470030] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self.driver.spawn(context, instance, image_meta, [ 809.470030] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 809.470030] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 809.470030] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 809.470030] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] vm_ref = self.build_virtual_machine(instance, [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] vif_infos = vmwarevif.get_vif_info(self._session, [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] for vif in network_info: [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] return self._sync_wrapper(fn, *args, **kwargs) [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self.wait() [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self[:] = self._gt.wait() [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] return self._exit_event.wait() [ 809.470441] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] result = hub.switch() [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] return self.greenlet.switch() [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] result = function(*args, **kwargs) [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] return func(*args, **kwargs) [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] raise e [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] nwinfo = self.network_api.allocate_for_instance( [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 809.470847] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] created_port_ids = self._update_ports_for_instance( [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] with excutils.save_and_reraise_exception(): [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self.force_reraise() [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] raise self.value [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] updated_port = self._update_port( [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] _ensure_no_port_binding_failure(port) [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 809.471269] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] raise exception.PortBindingFailed(port_id=port['id']) [ 809.471661] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] nova.exception.PortBindingFailed: Binding failed for port c825c2c5-b249-4fe4-9844-78dcf55b9a08, please check neutron logs for more information. [ 809.471661] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] [ 809.471661] env[63593]: INFO nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Terminating instance [ 809.471661] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Acquiring lock "refresh_cache-937998a3-dac3-4ce5-b363-7ceee313531a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 809.471791] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Acquired lock "refresh_cache-937998a3-dac3-4ce5-b363-7ceee313531a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 809.472031] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 809.511564] env[63593]: DEBUG nova.network.neutron [-] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 809.809978] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.421s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 809.809978] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 809.812252] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.013s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 809.988957] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 810.014159] env[63593]: INFO nova.compute.manager [-] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Took 1.02 seconds to deallocate network for instance. [ 810.016308] env[63593]: DEBUG nova.compute.claims [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 810.016492] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 810.060605] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 810.317024] env[63593]: DEBUG nova.compute.utils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 810.322019] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 810.322019] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 810.390046] env[63593]: DEBUG nova.policy [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '818839f731ae4d27bf16da5f17dd98be', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '74ad0f3f605a464b9dc378f81797700b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 810.563811] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Releasing lock "refresh_cache-937998a3-dac3-4ce5-b363-7ceee313531a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 810.564240] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 810.564433] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 810.566678] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3e98f335-2318-42eb-a54f-cb9793b3424a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.575837] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68c6ff85-9e3a-4389-b4d8-348a35aca321 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.598698] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 937998a3-dac3-4ce5-b363-7ceee313531a could not be found. [ 810.598924] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 810.599123] env[63593]: INFO nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Took 0.03 seconds to destroy the instance on the hypervisor. [ 810.599356] env[63593]: DEBUG oslo.service.loopingcall [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 810.601522] env[63593]: DEBUG nova.compute.manager [-] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 810.601626] env[63593]: DEBUG nova.network.neutron [-] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 810.619528] env[63593]: DEBUG nova.network.neutron [-] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 810.634086] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2adaa552-ab92-44e2-90be-ae91ab36d6db {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.643629] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11730c76-8527-40d9-b2ab-c3b144a29445 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.675465] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10554a45-9967-4994-9ca3-dab27a21e101 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.683271] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22e04726-1165-42ca-964c-d49af1515ec0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.697891] env[63593]: DEBUG nova.compute.provider_tree [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 810.745153] env[63593]: DEBUG nova.compute.manager [req-bf8d16a1-f8a6-415e-8f0a-b8485313308a req-368dd4ac-cce2-422c-962f-91cc1571d52c service nova] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Received event network-changed-c825c2c5-b249-4fe4-9844-78dcf55b9a08 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 810.745153] env[63593]: DEBUG nova.compute.manager [req-bf8d16a1-f8a6-415e-8f0a-b8485313308a req-368dd4ac-cce2-422c-962f-91cc1571d52c service nova] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Refreshing instance network info cache due to event network-changed-c825c2c5-b249-4fe4-9844-78dcf55b9a08. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 810.745153] env[63593]: DEBUG oslo_concurrency.lockutils [req-bf8d16a1-f8a6-415e-8f0a-b8485313308a req-368dd4ac-cce2-422c-962f-91cc1571d52c service nova] Acquiring lock "refresh_cache-937998a3-dac3-4ce5-b363-7ceee313531a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 810.745297] env[63593]: DEBUG oslo_concurrency.lockutils [req-bf8d16a1-f8a6-415e-8f0a-b8485313308a req-368dd4ac-cce2-422c-962f-91cc1571d52c service nova] Acquired lock "refresh_cache-937998a3-dac3-4ce5-b363-7ceee313531a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 810.745458] env[63593]: DEBUG nova.network.neutron [req-bf8d16a1-f8a6-415e-8f0a-b8485313308a req-368dd4ac-cce2-422c-962f-91cc1571d52c service nova] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Refreshing network info cache for port c825c2c5-b249-4fe4-9844-78dcf55b9a08 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 810.765600] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Successfully created port: 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 810.827516] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 811.125732] env[63593]: DEBUG nova.network.neutron [-] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 811.201662] env[63593]: DEBUG nova.scheduler.client.report [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 811.266897] env[63593]: DEBUG nova.network.neutron [req-bf8d16a1-f8a6-415e-8f0a-b8485313308a req-368dd4ac-cce2-422c-962f-91cc1571d52c service nova] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 811.398497] env[63593]: DEBUG nova.network.neutron [req-bf8d16a1-f8a6-415e-8f0a-b8485313308a req-368dd4ac-cce2-422c-962f-91cc1571d52c service nova] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 811.629097] env[63593]: INFO nova.compute.manager [-] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Took 1.03 seconds to deallocate network for instance. [ 811.630864] env[63593]: DEBUG nova.compute.claims [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 811.630999] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 811.646482] env[63593]: ERROR nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245, please check neutron logs for more information. [ 811.646482] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 811.646482] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 811.646482] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 811.646482] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 811.646482] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 811.646482] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 811.646482] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 811.646482] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 811.646482] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 811.646482] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 811.646482] env[63593]: ERROR nova.compute.manager raise self.value [ 811.646482] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 811.646482] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 811.646482] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 811.646482] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 811.647007] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 811.647007] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 811.647007] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245, please check neutron logs for more information. [ 811.647007] env[63593]: ERROR nova.compute.manager [ 811.647007] env[63593]: Traceback (most recent call last): [ 811.647007] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 811.647007] env[63593]: listener.cb(fileno) [ 811.647007] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 811.647007] env[63593]: result = function(*args, **kwargs) [ 811.647007] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 811.647007] env[63593]: return func(*args, **kwargs) [ 811.647007] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 811.647007] env[63593]: raise e [ 811.647007] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 811.647007] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 811.647007] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 811.647007] env[63593]: created_port_ids = self._update_ports_for_instance( [ 811.647007] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 811.647007] env[63593]: with excutils.save_and_reraise_exception(): [ 811.647007] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 811.647007] env[63593]: self.force_reraise() [ 811.647007] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 811.647007] env[63593]: raise self.value [ 811.647007] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 811.647007] env[63593]: updated_port = self._update_port( [ 811.647007] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 811.647007] env[63593]: _ensure_no_port_binding_failure(port) [ 811.647007] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 811.647007] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 811.647930] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245, please check neutron logs for more information. [ 811.647930] env[63593]: Removing descriptor: 18 [ 811.710091] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.898s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 811.710716] env[63593]: ERROR nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1bc61261-aadf-46a0-af53-6ec5c2632518, please check neutron logs for more information. [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Traceback (most recent call last): [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self.driver.spawn(context, instance, image_meta, [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self._vmops.spawn(context, instance, image_meta, injected_files, [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] vm_ref = self.build_virtual_machine(instance, [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] vif_infos = vmwarevif.get_vif_info(self._session, [ 811.710716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] for vif in network_info: [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] return self._sync_wrapper(fn, *args, **kwargs) [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self.wait() [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self[:] = self._gt.wait() [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] return self._exit_event.wait() [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] current.throw(*self._exc) [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 811.711049] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] result = function(*args, **kwargs) [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] return func(*args, **kwargs) [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] raise e [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] nwinfo = self.network_api.allocate_for_instance( [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] created_port_ids = self._update_ports_for_instance( [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] with excutils.save_and_reraise_exception(): [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] self.force_reraise() [ 811.711382] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 811.711716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] raise self.value [ 811.711716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 811.711716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] updated_port = self._update_port( [ 811.711716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 811.711716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] _ensure_no_port_binding_failure(port) [ 811.711716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 811.711716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] raise exception.PortBindingFailed(port_id=port['id']) [ 811.711716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] nova.exception.PortBindingFailed: Binding failed for port 1bc61261-aadf-46a0-af53-6ec5c2632518, please check neutron logs for more information. [ 811.711716] env[63593]: ERROR nova.compute.manager [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] [ 811.711716] env[63593]: DEBUG nova.compute.utils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Binding failed for port 1bc61261-aadf-46a0-af53-6ec5c2632518, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 811.712570] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.560s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 811.715644] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Build of instance 9bc65ce3-6459-4356-8a63-74f55b6f3b47 was re-scheduled: Binding failed for port 1bc61261-aadf-46a0-af53-6ec5c2632518, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 811.715985] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 811.716227] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "refresh_cache-9bc65ce3-6459-4356-8a63-74f55b6f3b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 811.716370] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquired lock "refresh_cache-9bc65ce3-6459-4356-8a63-74f55b6f3b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 811.716731] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 811.833955] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 811.858291] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 811.858572] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 811.858733] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 811.858909] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 811.859069] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 811.859221] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 811.859426] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 811.859580] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 811.859802] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 811.859988] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 811.860174] env[63593]: DEBUG nova.virt.hardware [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 811.861028] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3c556fd-12a0-423e-97d0-e86736da8a27 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 811.869398] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f09fbc4-5f78-4f57-b3a3-82f98842d4e5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 811.885395] env[63593]: ERROR nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245, please check neutron logs for more information. [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Traceback (most recent call last): [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] yield resources [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self.driver.spawn(context, instance, image_meta, [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] vm_ref = self.build_virtual_machine(instance, [ 811.885395] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] vif_infos = vmwarevif.get_vif_info(self._session, [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] for vif in network_info: [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] return self._sync_wrapper(fn, *args, **kwargs) [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self.wait() [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self[:] = self._gt.wait() [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] return self._exit_event.wait() [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 811.885769] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] current.throw(*self._exc) [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] result = function(*args, **kwargs) [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] return func(*args, **kwargs) [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] raise e [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] nwinfo = self.network_api.allocate_for_instance( [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] created_port_ids = self._update_ports_for_instance( [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] with excutils.save_and_reraise_exception(): [ 811.886127] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self.force_reraise() [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] raise self.value [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] updated_port = self._update_port( [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] _ensure_no_port_binding_failure(port) [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] raise exception.PortBindingFailed(port_id=port['id']) [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] nova.exception.PortBindingFailed: Binding failed for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245, please check neutron logs for more information. [ 811.886538] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] [ 811.886538] env[63593]: INFO nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Terminating instance [ 811.888040] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Acquiring lock "refresh_cache-b5889795-9995-4211-8b3f-8d14680fde6f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 811.888144] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Acquired lock "refresh_cache-b5889795-9995-4211-8b3f-8d14680fde6f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 811.888239] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 811.901807] env[63593]: DEBUG oslo_concurrency.lockutils [req-bf8d16a1-f8a6-415e-8f0a-b8485313308a req-368dd4ac-cce2-422c-962f-91cc1571d52c service nova] Releasing lock "refresh_cache-937998a3-dac3-4ce5-b363-7ceee313531a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 811.901965] env[63593]: DEBUG nova.compute.manager [req-bf8d16a1-f8a6-415e-8f0a-b8485313308a req-368dd4ac-cce2-422c-962f-91cc1571d52c service nova] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Received event network-vif-deleted-c825c2c5-b249-4fe4-9844-78dcf55b9a08 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 812.233829] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 812.281636] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 812.410149] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 812.472143] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 812.482171] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-631822a3-f57f-42c8-a10e-566b94828a7a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.490254] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ed7048a-74e3-4a02-a449-8a50bc35a3c9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.519884] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6bb64c1-11fd-4ad5-a112-9e32867342bc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.527076] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd6cec30-8e5a-4627-816d-9ea475a7cad1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.540008] env[63593]: DEBUG nova.compute.provider_tree [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 812.776848] env[63593]: DEBUG nova.compute.manager [req-d730990c-a2dd-4a06-a699-779e6875affe req-692ad015-6159-44f7-a1bc-89b2dac88e9d service nova] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Received event network-changed-4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 812.777063] env[63593]: DEBUG nova.compute.manager [req-d730990c-a2dd-4a06-a699-779e6875affe req-692ad015-6159-44f7-a1bc-89b2dac88e9d service nova] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Refreshing instance network info cache due to event network-changed-4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 812.777260] env[63593]: DEBUG oslo_concurrency.lockutils [req-d730990c-a2dd-4a06-a699-779e6875affe req-692ad015-6159-44f7-a1bc-89b2dac88e9d service nova] Acquiring lock "refresh_cache-b5889795-9995-4211-8b3f-8d14680fde6f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 812.783892] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Releasing lock "refresh_cache-9bc65ce3-6459-4356-8a63-74f55b6f3b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 812.784125] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 812.784290] env[63593]: DEBUG nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 812.784453] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 812.800793] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 812.974725] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Releasing lock "refresh_cache-b5889795-9995-4211-8b3f-8d14680fde6f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 812.975204] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 812.975397] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 812.975754] env[63593]: DEBUG oslo_concurrency.lockutils [req-d730990c-a2dd-4a06-a699-779e6875affe req-692ad015-6159-44f7-a1bc-89b2dac88e9d service nova] Acquired lock "refresh_cache-b5889795-9995-4211-8b3f-8d14680fde6f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 812.975925] env[63593]: DEBUG nova.network.neutron [req-d730990c-a2dd-4a06-a699-779e6875affe req-692ad015-6159-44f7-a1bc-89b2dac88e9d service nova] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Refreshing network info cache for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 812.977027] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-06019826-61c0-4ccb-9808-c4b1ca349022 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 812.986633] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51a439bf-c35d-4a21-98c0-5e6ab88d5628 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.007516] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b5889795-9995-4211-8b3f-8d14680fde6f could not be found. [ 813.007724] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 813.007907] env[63593]: INFO nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Took 0.03 seconds to destroy the instance on the hypervisor. [ 813.008166] env[63593]: DEBUG oslo.service.loopingcall [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 813.008370] env[63593]: DEBUG nova.compute.manager [-] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 813.008463] env[63593]: DEBUG nova.network.neutron [-] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 813.023669] env[63593]: DEBUG nova.network.neutron [-] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 813.042922] env[63593]: DEBUG nova.scheduler.client.report [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 813.302977] env[63593]: DEBUG nova.network.neutron [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 813.501142] env[63593]: DEBUG nova.network.neutron [req-d730990c-a2dd-4a06-a699-779e6875affe req-692ad015-6159-44f7-a1bc-89b2dac88e9d service nova] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 813.529398] env[63593]: DEBUG nova.network.neutron [-] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 813.548823] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.835s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 813.548823] env[63593]: ERROR nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 27db0986-3363-4052-bbc7-82e184d28792, please check neutron logs for more information. [ 813.548823] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] Traceback (most recent call last): [ 813.548823] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 813.548823] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self.driver.spawn(context, instance, image_meta, [ 813.548823] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 813.548823] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 813.548823] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 813.548823] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] vm_ref = self.build_virtual_machine(instance, [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] vif_infos = vmwarevif.get_vif_info(self._session, [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] for vif in network_info: [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] return self._sync_wrapper(fn, *args, **kwargs) [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self.wait() [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self[:] = self._gt.wait() [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] return self._exit_event.wait() [ 813.549165] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] current.throw(*self._exc) [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] result = function(*args, **kwargs) [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] return func(*args, **kwargs) [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] raise e [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] nwinfo = self.network_api.allocate_for_instance( [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] created_port_ids = self._update_ports_for_instance( [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 813.549602] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] with excutils.save_and_reraise_exception(): [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] self.force_reraise() [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] raise self.value [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] updated_port = self._update_port( [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] _ensure_no_port_binding_failure(port) [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] raise exception.PortBindingFailed(port_id=port['id']) [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] nova.exception.PortBindingFailed: Binding failed for port 27db0986-3363-4052-bbc7-82e184d28792, please check neutron logs for more information. [ 813.549978] env[63593]: ERROR nova.compute.manager [instance: b62cc771-973a-4d23-87b4-825211494d9c] [ 813.550391] env[63593]: DEBUG nova.compute.utils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Binding failed for port 27db0986-3363-4052-bbc7-82e184d28792, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 813.550391] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.895s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 813.553468] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Build of instance b62cc771-973a-4d23-87b4-825211494d9c was re-scheduled: Binding failed for port 27db0986-3363-4052-bbc7-82e184d28792, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 813.553706] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 813.553933] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquiring lock "refresh_cache-b62cc771-973a-4d23-87b4-825211494d9c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 813.554152] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Acquired lock "refresh_cache-b62cc771-973a-4d23-87b4-825211494d9c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 813.554262] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 813.558321] env[63593]: DEBUG nova.network.neutron [req-d730990c-a2dd-4a06-a699-779e6875affe req-692ad015-6159-44f7-a1bc-89b2dac88e9d service nova] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 813.806034] env[63593]: INFO nova.compute.manager [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: 9bc65ce3-6459-4356-8a63-74f55b6f3b47] Took 1.02 seconds to deallocate network for instance. [ 814.032111] env[63593]: INFO nova.compute.manager [-] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Took 1.02 seconds to deallocate network for instance. [ 814.034487] env[63593]: DEBUG nova.compute.claims [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 814.034717] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 814.061235] env[63593]: DEBUG oslo_concurrency.lockutils [req-d730990c-a2dd-4a06-a699-779e6875affe req-692ad015-6159-44f7-a1bc-89b2dac88e9d service nova] Releasing lock "refresh_cache-b5889795-9995-4211-8b3f-8d14680fde6f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 814.061469] env[63593]: DEBUG nova.compute.manager [req-d730990c-a2dd-4a06-a699-779e6875affe req-692ad015-6159-44f7-a1bc-89b2dac88e9d service nova] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Received event network-vif-deleted-4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 814.077102] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 814.132391] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 814.314935] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebd134f1-b57c-4d5b-9415-c32f19de57a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.322139] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b5c4c86-b991-4a26-8978-226b307b95a7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.353491] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db28dc9c-b8ff-4107-89ef-086838394b49 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.360966] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-975521fb-5ef3-4151-8fd3-7d668a69b4ef {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 814.374217] env[63593]: DEBUG nova.compute.provider_tree [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 814.635383] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Releasing lock "refresh_cache-b62cc771-973a-4d23-87b4-825211494d9c" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 814.636575] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 814.636862] env[63593]: DEBUG nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 814.637162] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 814.653984] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 814.833442] env[63593]: INFO nova.scheduler.client.report [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Deleted allocations for instance 9bc65ce3-6459-4356-8a63-74f55b6f3b47 [ 814.877154] env[63593]: DEBUG nova.scheduler.client.report [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 815.157264] env[63593]: DEBUG nova.network.neutron [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 815.346707] env[63593]: DEBUG oslo_concurrency.lockutils [None req-17629cad-cdd5-4f45-9447-b7cc45d8a0a4 tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "9bc65ce3-6459-4356-8a63-74f55b6f3b47" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 142.055s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.381425] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.831s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.382056] env[63593]: ERROR nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port eb902f6f-50ca-4b50-9958-abbdc4057646, please check neutron logs for more information. [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] Traceback (most recent call last): [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self.driver.spawn(context, instance, image_meta, [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self._vmops.spawn(context, instance, image_meta, injected_files, [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] vm_ref = self.build_virtual_machine(instance, [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] vif_infos = vmwarevif.get_vif_info(self._session, [ 815.382056] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] for vif in network_info: [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] return self._sync_wrapper(fn, *args, **kwargs) [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self.wait() [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self[:] = self._gt.wait() [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] return self._exit_event.wait() [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] current.throw(*self._exc) [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 815.382439] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] result = function(*args, **kwargs) [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] return func(*args, **kwargs) [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] raise e [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] nwinfo = self.network_api.allocate_for_instance( [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] created_port_ids = self._update_ports_for_instance( [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] with excutils.save_and_reraise_exception(): [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] self.force_reraise() [ 815.382861] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 815.383250] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] raise self.value [ 815.383250] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 815.383250] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] updated_port = self._update_port( [ 815.383250] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 815.383250] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] _ensure_no_port_binding_failure(port) [ 815.383250] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 815.383250] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] raise exception.PortBindingFailed(port_id=port['id']) [ 815.383250] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] nova.exception.PortBindingFailed: Binding failed for port eb902f6f-50ca-4b50-9958-abbdc4057646, please check neutron logs for more information. [ 815.383250] env[63593]: ERROR nova.compute.manager [instance: b965361a-6b76-4836-9887-b59ca990fd67] [ 815.383250] env[63593]: DEBUG nova.compute.utils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Binding failed for port eb902f6f-50ca-4b50-9958-abbdc4057646, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 815.383940] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.534s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.386726] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Build of instance b965361a-6b76-4836-9887-b59ca990fd67 was re-scheduled: Binding failed for port eb902f6f-50ca-4b50-9958-abbdc4057646, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 815.387154] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 815.388031] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "refresh_cache-b965361a-6b76-4836-9887-b59ca990fd67" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 815.388031] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquired lock "refresh_cache-b965361a-6b76-4836-9887-b59ca990fd67" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 815.388031] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 815.659934] env[63593]: INFO nova.compute.manager [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] [instance: b62cc771-973a-4d23-87b4-825211494d9c] Took 1.02 seconds to deallocate network for instance. [ 815.849890] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 815.915385] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 816.012122] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 816.155772] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4226c60-5a8b-4c64-815a-3539e560cf6f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.163015] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5be0f4c-add2-4a82-a0e4-d808bc999ab1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.196147] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56473893-8387-42eb-9b8e-cb6803c74d20 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.203697] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f86d320-cfbf-4623-ae16-c4a75588c43d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.217146] env[63593]: DEBUG nova.compute.provider_tree [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 816.368185] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.516521] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Releasing lock "refresh_cache-b965361a-6b76-4836-9887-b59ca990fd67" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 816.516759] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 816.516937] env[63593]: DEBUG nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 816.517117] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 816.530907] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 816.686815] env[63593]: INFO nova.scheduler.client.report [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Deleted allocations for instance b62cc771-973a-4d23-87b4-825211494d9c [ 816.719844] env[63593]: DEBUG nova.scheduler.client.report [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 817.033561] env[63593]: DEBUG nova.network.neutron [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 817.197930] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b6eb4717-8d02-4848-810b-c0955b7c0dce tempest-ListImageFiltersTestJSON-1010246967 tempest-ListImageFiltersTestJSON-1010246967-project-member] Lock "b62cc771-973a-4d23-87b4-825211494d9c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 143.062s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.225176] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.841s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.225806] env[63593]: ERROR nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 5b08c428-b00e-4a50-856f-c8d3925c051f, please check neutron logs for more information. [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Traceback (most recent call last): [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self.driver.spawn(context, instance, image_meta, [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self._vmops.spawn(context, instance, image_meta, injected_files, [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] vm_ref = self.build_virtual_machine(instance, [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] vif_infos = vmwarevif.get_vif_info(self._session, [ 817.225806] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] for vif in network_info: [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] return self._sync_wrapper(fn, *args, **kwargs) [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self.wait() [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self[:] = self._gt.wait() [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] return self._exit_event.wait() [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] result = hub.switch() [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 817.226197] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] return self.greenlet.switch() [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] result = function(*args, **kwargs) [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] return func(*args, **kwargs) [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] raise e [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] nwinfo = self.network_api.allocate_for_instance( [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] created_port_ids = self._update_ports_for_instance( [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] with excutils.save_and_reraise_exception(): [ 817.226591] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] self.force_reraise() [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] raise self.value [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] updated_port = self._update_port( [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] _ensure_no_port_binding_failure(port) [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] raise exception.PortBindingFailed(port_id=port['id']) [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] nova.exception.PortBindingFailed: Binding failed for port 5b08c428-b00e-4a50-856f-c8d3925c051f, please check neutron logs for more information. [ 817.226984] env[63593]: ERROR nova.compute.manager [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] [ 817.227341] env[63593]: DEBUG nova.compute.utils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Binding failed for port 5b08c428-b00e-4a50-856f-c8d3925c051f, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 817.227561] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 15.581s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.227768] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.227915] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 817.228205] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.585s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.230874] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Build of instance 077acb40-d01b-42ad-aa68-cbffc4087100 was re-scheduled: Binding failed for port 5b08c428-b00e-4a50-856f-c8d3925c051f, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 817.231295] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 817.231517] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "refresh_cache-077acb40-d01b-42ad-aa68-cbffc4087100" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 817.231661] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquired lock "refresh_cache-077acb40-d01b-42ad-aa68-cbffc4087100" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 817.231813] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 817.233171] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f220735-ca99-4c04-9894-73f5ecf13091 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.242139] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-577a8199-34c7-4323-a4ea-fc5388c634e2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.255650] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adaa30d1-8afd-4aee-9c6b-97a6d17a2b6e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.262670] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17cb1306-229c-41ac-b540-062a5ac6c6f2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.292309] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181492MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 817.292466] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 817.536088] env[63593]: INFO nova.compute.manager [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: b965361a-6b76-4836-9887-b59ca990fd67] Took 1.02 seconds to deallocate network for instance. [ 817.700452] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 817.753137] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 817.859017] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 817.990688] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb1fa7d6-181a-479a-8e02-35a3f25d13e9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.998360] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4149f0db-efb0-4d4b-bbf5-218c7c2a5475 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.029592] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aab5f2af-e1b5-4fdc-a3c4-7459185cfc96 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.036916] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a68b8e90-ab97-4bd4-9639-8c2d0bb89de0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.053718] env[63593]: DEBUG nova.compute.provider_tree [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 818.218650] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 818.363327] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Releasing lock "refresh_cache-077acb40-d01b-42ad-aa68-cbffc4087100" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 818.363581] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 818.363754] env[63593]: DEBUG nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 818.363925] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 818.388675] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 818.556978] env[63593]: DEBUG nova.scheduler.client.report [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 818.569271] env[63593]: INFO nova.scheduler.client.report [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Deleted allocations for instance b965361a-6b76-4836-9887-b59ca990fd67 [ 818.891616] env[63593]: DEBUG nova.network.neutron [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 819.061566] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.833s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 819.062223] env[63593]: ERROR nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6, please check neutron logs for more information. [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Traceback (most recent call last): [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self.driver.spawn(context, instance, image_meta, [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] vm_ref = self.build_virtual_machine(instance, [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] vif_infos = vmwarevif.get_vif_info(self._session, [ 819.062223] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] for vif in network_info: [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] return self._sync_wrapper(fn, *args, **kwargs) [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self.wait() [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self[:] = self._gt.wait() [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] return self._exit_event.wait() [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] current.throw(*self._exc) [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 819.062559] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] result = function(*args, **kwargs) [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] return func(*args, **kwargs) [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] raise e [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] nwinfo = self.network_api.allocate_for_instance( [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] created_port_ids = self._update_ports_for_instance( [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] with excutils.save_and_reraise_exception(): [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] self.force_reraise() [ 819.062957] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 819.063354] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] raise self.value [ 819.063354] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 819.063354] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] updated_port = self._update_port( [ 819.063354] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 819.063354] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] _ensure_no_port_binding_failure(port) [ 819.063354] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 819.063354] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] raise exception.PortBindingFailed(port_id=port['id']) [ 819.063354] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] nova.exception.PortBindingFailed: Binding failed for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6, please check neutron logs for more information. [ 819.063354] env[63593]: ERROR nova.compute.manager [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] [ 819.063354] env[63593]: DEBUG nova.compute.utils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Binding failed for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 819.064097] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.122s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 819.066842] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Build of instance 1db3e15b-3168-4d01-930f-fe8008b95df6 was re-scheduled: Binding failed for port d4d842d7-7eaa-4dbc-8da8-96e951e618c6, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 819.067249] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 819.067467] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquiring lock "refresh_cache-1db3e15b-3168-4d01-930f-fe8008b95df6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 819.067611] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Acquired lock "refresh_cache-1db3e15b-3168-4d01-930f-fe8008b95df6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 819.067828] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 819.076284] env[63593]: DEBUG oslo_concurrency.lockutils [None req-95d60222-7236-4080-8f10-d5a2ed28196e tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "b965361a-6b76-4836-9887-b59ca990fd67" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 144.020s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 819.397379] env[63593]: INFO nova.compute.manager [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 077acb40-d01b-42ad-aa68-cbffc4087100] Took 1.03 seconds to deallocate network for instance. [ 819.578969] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 819.588422] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 819.715230] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 819.841223] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ead3b229-6fe7-4094-baa5-628cd3972cdd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.850397] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcb6f20e-2abe-4db8-9480-57e0937ea7c5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.886220] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91d85140-b827-4a13-acff-bda674432269 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.890785] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3af1585-39e4-4584-81e4-57e5e39f8bb6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 819.915019] env[63593]: DEBUG nova.compute.provider_tree [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 820.098931] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.217520] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Releasing lock "refresh_cache-1db3e15b-3168-4d01-930f-fe8008b95df6" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 820.218118] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 820.218519] env[63593]: DEBUG nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 820.218764] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 820.248733] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 820.418007] env[63593]: DEBUG nova.scheduler.client.report [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 820.457429] env[63593]: INFO nova.scheduler.client.report [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Deleted allocations for instance 077acb40-d01b-42ad-aa68-cbffc4087100 [ 820.634238] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "fbd620a6-3743-4737-98b7-b13928b6587d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.634526] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "fbd620a6-3743-4737-98b7-b13928b6587d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.755403] env[63593]: DEBUG nova.network.neutron [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 820.925218] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.861s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.925881] env[63593]: ERROR nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 399af546-2adb-454f-b248-8377b08a6d41, please check neutron logs for more information. [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Traceback (most recent call last): [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self.driver.spawn(context, instance, image_meta, [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self._vmops.spawn(context, instance, image_meta, injected_files, [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] vm_ref = self.build_virtual_machine(instance, [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] vif_infos = vmwarevif.get_vif_info(self._session, [ 820.925881] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] for vif in network_info: [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] return self._sync_wrapper(fn, *args, **kwargs) [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self.wait() [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self[:] = self._gt.wait() [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] return self._exit_event.wait() [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] current.throw(*self._exc) [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 820.926220] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] result = function(*args, **kwargs) [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] return func(*args, **kwargs) [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] raise e [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] nwinfo = self.network_api.allocate_for_instance( [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] created_port_ids = self._update_ports_for_instance( [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] with excutils.save_and_reraise_exception(): [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] self.force_reraise() [ 820.926588] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 820.926955] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] raise self.value [ 820.926955] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 820.926955] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] updated_port = self._update_port( [ 820.926955] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 820.926955] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] _ensure_no_port_binding_failure(port) [ 820.926955] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 820.926955] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] raise exception.PortBindingFailed(port_id=port['id']) [ 820.926955] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] nova.exception.PortBindingFailed: Binding failed for port 399af546-2adb-454f-b248-8377b08a6d41, please check neutron logs for more information. [ 820.926955] env[63593]: ERROR nova.compute.manager [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] [ 820.926955] env[63593]: DEBUG nova.compute.utils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Binding failed for port 399af546-2adb-454f-b248-8377b08a6d41, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 820.930682] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.584s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.931347] env[63593]: INFO nova.compute.claims [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 820.933961] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Build of instance b27b2ed8-ec99-4962-8742-3a8264906b47 was re-scheduled: Binding failed for port 399af546-2adb-454f-b248-8377b08a6d41, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 820.934406] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 820.934630] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Acquiring lock "refresh_cache-b27b2ed8-ec99-4962-8742-3a8264906b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 820.934780] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Acquired lock "refresh_cache-b27b2ed8-ec99-4962-8742-3a8264906b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 820.934931] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 820.969261] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3a1a0253-48fb-4a99-a2f7-4a29510fe7b8 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "077acb40-d01b-42ad-aa68-cbffc4087100" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 145.831s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.257991] env[63593]: INFO nova.compute.manager [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] [instance: 1db3e15b-3168-4d01-930f-fe8008b95df6] Took 1.04 seconds to deallocate network for instance. [ 821.473920] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 821.475027] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 821.576782] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 821.997164] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 822.080919] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Releasing lock "refresh_cache-b27b2ed8-ec99-4962-8742-3a8264906b47" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 822.081173] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 822.081354] env[63593]: DEBUG nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 822.081517] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 822.102680] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 822.265674] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1a7abac-84ba-4dda-bf11-c3ef0f9f3a3f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.276171] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7493762-587f-49e9-897b-815d160233e1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.316152] env[63593]: INFO nova.scheduler.client.report [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Deleted allocations for instance 1db3e15b-3168-4d01-930f-fe8008b95df6 [ 822.322249] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88b7e4f4-9a2a-4f77-94ea-6b860fceb9e0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.329350] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f905205f-3940-4abb-b0df-6a8cd8a4a63d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 822.345858] env[63593]: DEBUG nova.compute.provider_tree [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 822.604853] env[63593]: DEBUG nova.network.neutron [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 822.825709] env[63593]: DEBUG oslo_concurrency.lockutils [None req-88637419-46da-4157-9ac0-de8bf4b8d065 tempest-SecurityGroupsTestJSON-1352724837 tempest-SecurityGroupsTestJSON-1352724837-project-member] Lock "1db3e15b-3168-4d01-930f-fe8008b95df6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 144.254s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 822.849518] env[63593]: DEBUG nova.scheduler.client.report [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 823.113180] env[63593]: INFO nova.compute.manager [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] [instance: b27b2ed8-ec99-4962-8742-3a8264906b47] Took 1.03 seconds to deallocate network for instance. [ 823.256821] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 823.257259] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 823.329553] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 823.354637] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.426s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 823.355248] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 823.358140] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.342s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 823.849744] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 823.865324] env[63593]: DEBUG nova.compute.utils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 823.867818] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 823.868185] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 823.944388] env[63593]: DEBUG nova.policy [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a33c7c30aafd411a9dd1418abb3a5f83', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'aa0c91fd0f8346fc853a1488bf6bb3f4', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 824.041872] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "328b91c6-4ae4-442f-ac87-519658a812ff" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.042157] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "328b91c6-4ae4-442f-ac87-519658a812ff" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.145384] env[63593]: INFO nova.scheduler.client.report [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Deleted allocations for instance b27b2ed8-ec99-4962-8742-3a8264906b47 [ 824.197457] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6294d6be-852a-4b68-b003-3c99465fe6d5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.207927] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-23f3499d-1c1e-4a04-9ecd-5f98b6f60a3f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.263838] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0dc359d-9f37-48e1-b12f-e204165e32da {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.275560] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-042ba58b-b9dc-40e7-a2e0-14a2a99e3c9b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.295033] env[63593]: DEBUG nova.compute.provider_tree [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 824.331076] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Successfully created port: 6a72110e-d211-42af-9cab-dc99d21308f2 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 824.368671] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 824.653745] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b708209e-9da8-4072-bc5e-83537e47e089 tempest-AttachInterfacesV270Test-903941174 tempest-AttachInterfacesV270Test-903941174-project-member] Lock "b27b2ed8-ec99-4962-8742-3a8264906b47" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 142.687s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 824.798635] env[63593]: DEBUG nova.scheduler.client.report [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 825.126148] env[63593]: DEBUG nova.compute.manager [req-28c195e2-bbdb-4bd4-a244-bbc873bb4a49 req-d7f5fb07-5ab6-4cd2-bd06-052192430708 service nova] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Received event network-changed-6a72110e-d211-42af-9cab-dc99d21308f2 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 825.126148] env[63593]: DEBUG nova.compute.manager [req-28c195e2-bbdb-4bd4-a244-bbc873bb4a49 req-d7f5fb07-5ab6-4cd2-bd06-052192430708 service nova] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Refreshing instance network info cache due to event network-changed-6a72110e-d211-42af-9cab-dc99d21308f2. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 825.126148] env[63593]: DEBUG oslo_concurrency.lockutils [req-28c195e2-bbdb-4bd4-a244-bbc873bb4a49 req-d7f5fb07-5ab6-4cd2-bd06-052192430708 service nova] Acquiring lock "refresh_cache-9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 825.126475] env[63593]: DEBUG oslo_concurrency.lockutils [req-28c195e2-bbdb-4bd4-a244-bbc873bb4a49 req-d7f5fb07-5ab6-4cd2-bd06-052192430708 service nova] Acquired lock "refresh_cache-9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 825.126773] env[63593]: DEBUG nova.network.neutron [req-28c195e2-bbdb-4bd4-a244-bbc873bb4a49 req-d7f5fb07-5ab6-4cd2-bd06-052192430708 service nova] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Refreshing network info cache for port 6a72110e-d211-42af-9cab-dc99d21308f2 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 825.158164] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 825.305442] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.947s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 825.306114] env[63593]: ERROR nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c5740fc0-200d-47ec-8497-d1e295793409, please check neutron logs for more information. [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Traceback (most recent call last): [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self.driver.spawn(context, instance, image_meta, [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] vm_ref = self.build_virtual_machine(instance, [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] vif_infos = vmwarevif.get_vif_info(self._session, [ 825.306114] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] for vif in network_info: [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] return self._sync_wrapper(fn, *args, **kwargs) [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self.wait() [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self[:] = self._gt.wait() [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] return self._exit_event.wait() [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] current.throw(*self._exc) [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 825.306496] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] result = function(*args, **kwargs) [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] return func(*args, **kwargs) [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] raise e [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] nwinfo = self.network_api.allocate_for_instance( [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] created_port_ids = self._update_ports_for_instance( [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] with excutils.save_and_reraise_exception(): [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] self.force_reraise() [ 825.306913] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 825.307328] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] raise self.value [ 825.307328] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 825.307328] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] updated_port = self._update_port( [ 825.307328] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 825.307328] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] _ensure_no_port_binding_failure(port) [ 825.307328] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 825.307328] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] raise exception.PortBindingFailed(port_id=port['id']) [ 825.307328] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] nova.exception.PortBindingFailed: Binding failed for port c5740fc0-200d-47ec-8497-d1e295793409, please check neutron logs for more information. [ 825.307328] env[63593]: ERROR nova.compute.manager [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] [ 825.307328] env[63593]: DEBUG nova.compute.utils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Binding failed for port c5740fc0-200d-47ec-8497-d1e295793409, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 825.308565] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.677s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 825.315027] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Build of instance a269ba6a-8439-448f-ad51-e3084c89c2a5 was re-scheduled: Binding failed for port c5740fc0-200d-47ec-8497-d1e295793409, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 825.315027] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 825.315027] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "refresh_cache-a269ba6a-8439-448f-ad51-e3084c89c2a5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 825.315027] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "refresh_cache-a269ba6a-8439-448f-ad51-e3084c89c2a5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 825.315262] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 825.378234] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 825.411071] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 825.411245] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 825.411397] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 825.411831] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 825.412090] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 825.413094] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 825.413094] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 825.413094] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 825.413094] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 825.413094] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 825.413299] env[63593]: DEBUG nova.virt.hardware [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 825.415910] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0378ab41-aeae-48c2-8c4e-b266678728d4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.427227] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e39b5b6-6f7b-4dfa-ab1b-139c10f1a548 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.477188] env[63593]: ERROR nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6a72110e-d211-42af-9cab-dc99d21308f2, please check neutron logs for more information. [ 825.477188] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 825.477188] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 825.477188] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 825.477188] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 825.477188] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 825.477188] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 825.477188] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 825.477188] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 825.477188] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 825.477188] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 825.477188] env[63593]: ERROR nova.compute.manager raise self.value [ 825.477188] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 825.477188] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 825.477188] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 825.477188] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 825.477767] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 825.477767] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 825.477767] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6a72110e-d211-42af-9cab-dc99d21308f2, please check neutron logs for more information. [ 825.477767] env[63593]: ERROR nova.compute.manager [ 825.477767] env[63593]: Traceback (most recent call last): [ 825.477767] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 825.477767] env[63593]: listener.cb(fileno) [ 825.477767] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 825.477767] env[63593]: result = function(*args, **kwargs) [ 825.477767] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 825.477767] env[63593]: return func(*args, **kwargs) [ 825.477767] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 825.477767] env[63593]: raise e [ 825.477767] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 825.477767] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 825.477767] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 825.477767] env[63593]: created_port_ids = self._update_ports_for_instance( [ 825.477767] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 825.477767] env[63593]: with excutils.save_and_reraise_exception(): [ 825.477767] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 825.477767] env[63593]: self.force_reraise() [ 825.477767] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 825.477767] env[63593]: raise self.value [ 825.477767] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 825.477767] env[63593]: updated_port = self._update_port( [ 825.477767] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 825.477767] env[63593]: _ensure_no_port_binding_failure(port) [ 825.477767] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 825.477767] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 825.478541] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 6a72110e-d211-42af-9cab-dc99d21308f2, please check neutron logs for more information. [ 825.478541] env[63593]: Removing descriptor: 18 [ 825.478541] env[63593]: ERROR nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6a72110e-d211-42af-9cab-dc99d21308f2, please check neutron logs for more information. [ 825.478541] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Traceback (most recent call last): [ 825.478541] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 825.478541] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] yield resources [ 825.478541] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 825.478541] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self.driver.spawn(context, instance, image_meta, [ 825.478541] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 825.478541] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 825.478541] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 825.478541] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] vm_ref = self.build_virtual_machine(instance, [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] vif_infos = vmwarevif.get_vif_info(self._session, [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] for vif in network_info: [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] return self._sync_wrapper(fn, *args, **kwargs) [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self.wait() [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self[:] = self._gt.wait() [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] return self._exit_event.wait() [ 825.478910] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] result = hub.switch() [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] return self.greenlet.switch() [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] result = function(*args, **kwargs) [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] return func(*args, **kwargs) [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] raise e [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] nwinfo = self.network_api.allocate_for_instance( [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 825.479268] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] created_port_ids = self._update_ports_for_instance( [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] with excutils.save_and_reraise_exception(): [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self.force_reraise() [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] raise self.value [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] updated_port = self._update_port( [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] _ensure_no_port_binding_failure(port) [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 825.479610] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] raise exception.PortBindingFailed(port_id=port['id']) [ 825.479944] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] nova.exception.PortBindingFailed: Binding failed for port 6a72110e-d211-42af-9cab-dc99d21308f2, please check neutron logs for more information. [ 825.479944] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] [ 825.479944] env[63593]: INFO nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Terminating instance [ 825.480818] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Acquiring lock "refresh_cache-9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 825.651986] env[63593]: DEBUG nova.network.neutron [req-28c195e2-bbdb-4bd4-a244-bbc873bb4a49 req-d7f5fb07-5ab6-4cd2-bd06-052192430708 service nova] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 825.682392] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 825.732857] env[63593]: DEBUG nova.network.neutron [req-28c195e2-bbdb-4bd4-a244-bbc873bb4a49 req-d7f5fb07-5ab6-4cd2-bd06-052192430708 service nova] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 825.834236] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 825.949071] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 826.140942] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-deb63caf-3f90-454e-aa25-cbed3112677e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.148666] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db98cfe7-eb5c-4972-8bb0-23a8b05328d5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.178423] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdae687a-7dcc-47cc-aa78-06ae7c11bbe7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.185623] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7099b393-428c-4479-ad65-10fe9bd4b163 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.203822] env[63593]: DEBUG nova.compute.provider_tree [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 826.234462] env[63593]: DEBUG oslo_concurrency.lockutils [req-28c195e2-bbdb-4bd4-a244-bbc873bb4a49 req-d7f5fb07-5ab6-4cd2-bd06-052192430708 service nova] Releasing lock "refresh_cache-9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 826.234633] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Acquired lock "refresh_cache-9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 826.234824] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 826.452502] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "refresh_cache-a269ba6a-8439-448f-ad51-e3084c89c2a5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 826.453034] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 826.453240] env[63593]: DEBUG nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 826.453772] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 826.475142] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 826.707188] env[63593]: DEBUG nova.scheduler.client.report [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 826.763020] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 826.823109] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 826.980471] env[63593]: DEBUG nova.network.neutron [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 827.175306] env[63593]: DEBUG nova.compute.manager [req-6fb809ae-4e08-4c4e-bcdf-3c3883614a43 req-cf872c8b-b1d9-41fd-b7ff-15fb27d8e104 service nova] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Received event network-vif-deleted-6a72110e-d211-42af-9cab-dc99d21308f2 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 827.214634] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.904s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 827.214634] env[63593]: ERROR nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c825c2c5-b249-4fe4-9844-78dcf55b9a08, please check neutron logs for more information. [ 827.214634] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Traceback (most recent call last): [ 827.214634] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 827.214634] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self.driver.spawn(context, instance, image_meta, [ 827.214634] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 827.214634] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 827.214634] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 827.214634] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] vm_ref = self.build_virtual_machine(instance, [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] vif_infos = vmwarevif.get_vif_info(self._session, [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] for vif in network_info: [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] return self._sync_wrapper(fn, *args, **kwargs) [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self.wait() [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self[:] = self._gt.wait() [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] return self._exit_event.wait() [ 827.215399] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] result = hub.switch() [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] return self.greenlet.switch() [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] result = function(*args, **kwargs) [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] return func(*args, **kwargs) [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] raise e [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] nwinfo = self.network_api.allocate_for_instance( [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 827.215903] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] created_port_ids = self._update_ports_for_instance( [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] with excutils.save_and_reraise_exception(): [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] self.force_reraise() [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] raise self.value [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] updated_port = self._update_port( [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] _ensure_no_port_binding_failure(port) [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 827.216302] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] raise exception.PortBindingFailed(port_id=port['id']) [ 827.216645] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] nova.exception.PortBindingFailed: Binding failed for port c825c2c5-b249-4fe4-9844-78dcf55b9a08, please check neutron logs for more information. [ 827.216645] env[63593]: ERROR nova.compute.manager [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] [ 827.216645] env[63593]: DEBUG nova.compute.utils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Binding failed for port c825c2c5-b249-4fe4-9844-78dcf55b9a08, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 827.223232] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Build of instance 937998a3-dac3-4ce5-b363-7ceee313531a was re-scheduled: Binding failed for port c825c2c5-b249-4fe4-9844-78dcf55b9a08, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 827.223232] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 827.223232] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Acquiring lock "refresh_cache-937998a3-dac3-4ce5-b363-7ceee313531a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 827.223232] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Acquired lock "refresh_cache-937998a3-dac3-4ce5-b363-7ceee313531a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 827.223447] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 827.223447] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.188s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.331738] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Releasing lock "refresh_cache-9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 827.332268] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 827.332471] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 827.332901] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d90f4ddd-d225-48b3-83c5-89804154b6d5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.342543] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-728900fc-1d85-485e-bfdf-65f4b8344a31 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 827.365999] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a could not be found. [ 827.366683] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 827.367041] env[63593]: INFO nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Took 0.03 seconds to destroy the instance on the hypervisor. [ 827.367520] env[63593]: DEBUG oslo.service.loopingcall [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 827.367956] env[63593]: DEBUG nova.compute.manager [-] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 827.368575] env[63593]: DEBUG nova.network.neutron [-] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 827.391794] env[63593]: DEBUG nova.network.neutron [-] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 827.482260] env[63593]: INFO nova.compute.manager [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: a269ba6a-8439-448f-ad51-e3084c89c2a5] Took 1.03 seconds to deallocate network for instance. [ 827.750427] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 827.895125] env[63593]: DEBUG nova.network.neutron [-] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 827.899745] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 828.081017] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54f8c590-8958-4c60-a387-10a6ce5a3537 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.091064] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5fd298f7-861b-4506-89d6-91f3108439ae {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.129174] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca49e935-eb9a-4c99-baed-99c3378f37bc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.136820] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6767acf6-0fba-4cd6-9ad6-5b64ea0a3953 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.151253] env[63593]: DEBUG nova.compute.provider_tree [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 828.401786] env[63593]: INFO nova.compute.manager [-] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Took 1.03 seconds to deallocate network for instance. [ 828.408019] env[63593]: DEBUG nova.compute.claims [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 828.408019] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 828.408019] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Releasing lock "refresh_cache-937998a3-dac3-4ce5-b363-7ceee313531a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 828.408019] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 828.408262] env[63593]: DEBUG nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 828.408262] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 828.426235] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 828.531081] env[63593]: INFO nova.scheduler.client.report [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Deleted allocations for instance a269ba6a-8439-448f-ad51-e3084c89c2a5 [ 828.654026] env[63593]: DEBUG nova.scheduler.client.report [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 828.930336] env[63593]: DEBUG nova.network.neutron [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 829.038821] env[63593]: DEBUG oslo_concurrency.lockutils [None req-15a15e33-a61f-476a-940f-d62a218bf4b1 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "a269ba6a-8439-448f-ad51-e3084c89c2a5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 144.852s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 829.160653] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.938s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 829.161289] env[63593]: ERROR nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245, please check neutron logs for more information. [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Traceback (most recent call last): [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self.driver.spawn(context, instance, image_meta, [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] vm_ref = self.build_virtual_machine(instance, [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] vif_infos = vmwarevif.get_vif_info(self._session, [ 829.161289] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] for vif in network_info: [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] return self._sync_wrapper(fn, *args, **kwargs) [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self.wait() [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self[:] = self._gt.wait() [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] return self._exit_event.wait() [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] current.throw(*self._exc) [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 829.161606] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] result = function(*args, **kwargs) [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] return func(*args, **kwargs) [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] raise e [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] nwinfo = self.network_api.allocate_for_instance( [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] created_port_ids = self._update_ports_for_instance( [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] with excutils.save_and_reraise_exception(): [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] self.force_reraise() [ 829.161951] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 829.162409] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] raise self.value [ 829.162409] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 829.162409] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] updated_port = self._update_port( [ 829.162409] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 829.162409] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] _ensure_no_port_binding_failure(port) [ 829.162409] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 829.162409] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] raise exception.PortBindingFailed(port_id=port['id']) [ 829.162409] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] nova.exception.PortBindingFailed: Binding failed for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245, please check neutron logs for more information. [ 829.162409] env[63593]: ERROR nova.compute.manager [instance: b5889795-9995-4211-8b3f-8d14680fde6f] [ 829.162409] env[63593]: DEBUG nova.compute.utils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Binding failed for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 829.163688] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.796s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 829.165213] env[63593]: INFO nova.compute.claims [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 829.169260] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Build of instance b5889795-9995-4211-8b3f-8d14680fde6f was re-scheduled: Binding failed for port 4a5ea8f9-6cdd-4ab3-9946-e1f2485f3245, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 829.169260] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 829.169260] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Acquiring lock "refresh_cache-b5889795-9995-4211-8b3f-8d14680fde6f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 829.169260] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Acquired lock "refresh_cache-b5889795-9995-4211-8b3f-8d14680fde6f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 829.169422] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 829.433953] env[63593]: INFO nova.compute.manager [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] [instance: 937998a3-dac3-4ce5-b363-7ceee313531a] Took 1.03 seconds to deallocate network for instance. [ 829.546518] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 829.696724] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 829.801897] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 830.072747] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.311542] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Releasing lock "refresh_cache-b5889795-9995-4211-8b3f-8d14680fde6f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 830.311542] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 830.311542] env[63593]: DEBUG nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 830.311542] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 830.336471] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 830.485964] env[63593]: INFO nova.scheduler.client.report [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Deleted allocations for instance 937998a3-dac3-4ce5-b363-7ceee313531a [ 830.559654] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-446480ad-a123-4fea-8cd6-1229dce56b9a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.568309] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea4d40fb-c810-4364-b06a-f58e90b9430f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.602030] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9824ccc7-2275-4ca1-86ae-271b5edd2b25 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.611264] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-516be028-63a0-4a3e-bfa2-238225d96c66 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.625212] env[63593]: DEBUG nova.compute.provider_tree [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 830.659115] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "b053676a-81b8-4345-b257-1d67976fd077" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.659352] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "b053676a-81b8-4345-b257-1d67976fd077" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.838199] env[63593]: DEBUG nova.network.neutron [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 830.994568] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f8ac2f59-c464-4625-9478-0cb6b914229b tempest-FloatingIPsAssociationNegativeTestJSON-1814401124 tempest-FloatingIPsAssociationNegativeTestJSON-1814401124-project-member] Lock "937998a3-dac3-4ce5-b363-7ceee313531a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 146.215s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.039289] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquiring lock "cf724d13-e34e-4a17-9b54-56190891d5db" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.039289] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "cf724d13-e34e-4a17-9b54-56190891d5db" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.133150] env[63593]: DEBUG nova.scheduler.client.report [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 831.345844] env[63593]: INFO nova.compute.manager [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: b5889795-9995-4211-8b3f-8d14680fde6f] Took 1.03 seconds to deallocate network for instance. [ 831.498919] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 831.638323] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "48e001ae-cc04-4d95-be43-2b2b3556c6db" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.638483] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.641519] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.478s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.642076] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 831.647322] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 14.353s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.673187] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.673187] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.708093] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "d760b552-e2ff-46db-88eb-f5f58fc88239" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.708637] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "d760b552-e2ff-46db-88eb-f5f58fc88239" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 832.026203] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 832.149447] env[63593]: DEBUG nova.compute.utils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 832.152324] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 832.152324] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 832.219764] env[63593]: DEBUG nova.policy [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd992216b789d40239f66c9855832867d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '883751f1569d42fba59e95751082398a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 832.392873] env[63593]: INFO nova.scheduler.client.report [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Deleted allocations for instance b5889795-9995-4211-8b3f-8d14680fde6f [ 832.654882] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 832.680773] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 832.681426] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 94486acf-7f7b-4187-b9d1-332f70488bd5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 832.780613] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Successfully created port: 075296e6-0c1e-4d55-a362-2fe91040eefb {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 832.902373] env[63593]: DEBUG oslo_concurrency.lockutils [None req-71141dc1-7e53-44b0-99dc-c34f166d23a4 tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Lock "b5889795-9995-4211-8b3f-8d14680fde6f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 145.623s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.184151] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 94ff0182-d1ce-405f-bdb5-a2399bd0f4de has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 833.408473] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 833.668474] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 833.691061] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e312fd5-01bf-4bd4-8a61-26624ae05036 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 833.703398] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 833.704029] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 833.704029] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 833.704029] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 833.704387] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 833.704387] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 833.707106] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 833.707256] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 833.707513] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 833.707630] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 833.708057] env[63593]: DEBUG nova.virt.hardware [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 833.708654] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a607feb-ee5b-4207-a170-3ff778baa2e2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 833.719333] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31c8d698-5299-4d24-a8b3-068cc3cd6b6d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 833.740732] env[63593]: DEBUG nova.compute.manager [req-fe688d54-8f0a-4c71-ad8b-5399ed306894 req-1fe4d861-8e2a-49de-a8ae-bff1b482c72d service nova] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Received event network-changed-075296e6-0c1e-4d55-a362-2fe91040eefb {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 833.740974] env[63593]: DEBUG nova.compute.manager [req-fe688d54-8f0a-4c71-ad8b-5399ed306894 req-1fe4d861-8e2a-49de-a8ae-bff1b482c72d service nova] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Refreshing instance network info cache due to event network-changed-075296e6-0c1e-4d55-a362-2fe91040eefb. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 833.741187] env[63593]: DEBUG oslo_concurrency.lockutils [req-fe688d54-8f0a-4c71-ad8b-5399ed306894 req-1fe4d861-8e2a-49de-a8ae-bff1b482c72d service nova] Acquiring lock "refresh_cache-94486acf-7f7b-4187-b9d1-332f70488bd5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 833.741262] env[63593]: DEBUG oslo_concurrency.lockutils [req-fe688d54-8f0a-4c71-ad8b-5399ed306894 req-1fe4d861-8e2a-49de-a8ae-bff1b482c72d service nova] Acquired lock "refresh_cache-94486acf-7f7b-4187-b9d1-332f70488bd5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 833.741521] env[63593]: DEBUG nova.network.neutron [req-fe688d54-8f0a-4c71-ad8b-5399ed306894 req-1fe4d861-8e2a-49de-a8ae-bff1b482c72d service nova] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Refreshing network info cache for port 075296e6-0c1e-4d55-a362-2fe91040eefb {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 833.897606] env[63593]: ERROR nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 075296e6-0c1e-4d55-a362-2fe91040eefb, please check neutron logs for more information. [ 833.897606] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 833.897606] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 833.897606] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 833.897606] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 833.897606] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 833.897606] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 833.897606] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 833.897606] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 833.897606] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 833.897606] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 833.897606] env[63593]: ERROR nova.compute.manager raise self.value [ 833.897606] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 833.897606] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 833.897606] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 833.897606] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 833.898104] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 833.898104] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 833.898104] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 075296e6-0c1e-4d55-a362-2fe91040eefb, please check neutron logs for more information. [ 833.898104] env[63593]: ERROR nova.compute.manager [ 833.898104] env[63593]: Traceback (most recent call last): [ 833.898104] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 833.898104] env[63593]: listener.cb(fileno) [ 833.898104] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 833.898104] env[63593]: result = function(*args, **kwargs) [ 833.898104] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 833.898104] env[63593]: return func(*args, **kwargs) [ 833.898104] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 833.898104] env[63593]: raise e [ 833.898104] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 833.898104] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 833.898104] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 833.898104] env[63593]: created_port_ids = self._update_ports_for_instance( [ 833.898104] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 833.898104] env[63593]: with excutils.save_and_reraise_exception(): [ 833.898104] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 833.898104] env[63593]: self.force_reraise() [ 833.898104] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 833.898104] env[63593]: raise self.value [ 833.898104] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 833.898104] env[63593]: updated_port = self._update_port( [ 833.898104] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 833.898104] env[63593]: _ensure_no_port_binding_failure(port) [ 833.898104] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 833.898104] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 833.898870] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 075296e6-0c1e-4d55-a362-2fe91040eefb, please check neutron logs for more information. [ 833.898870] env[63593]: Removing descriptor: 18 [ 833.898870] env[63593]: ERROR nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 075296e6-0c1e-4d55-a362-2fe91040eefb, please check neutron logs for more information. [ 833.898870] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Traceback (most recent call last): [ 833.898870] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 833.898870] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] yield resources [ 833.898870] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 833.898870] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self.driver.spawn(context, instance, image_meta, [ 833.898870] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 833.898870] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 833.898870] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 833.898870] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] vm_ref = self.build_virtual_machine(instance, [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] vif_infos = vmwarevif.get_vif_info(self._session, [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] for vif in network_info: [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] return self._sync_wrapper(fn, *args, **kwargs) [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self.wait() [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self[:] = self._gt.wait() [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] return self._exit_event.wait() [ 833.899231] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] result = hub.switch() [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] return self.greenlet.switch() [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] result = function(*args, **kwargs) [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] return func(*args, **kwargs) [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] raise e [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] nwinfo = self.network_api.allocate_for_instance( [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 833.899588] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] created_port_ids = self._update_ports_for_instance( [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] with excutils.save_and_reraise_exception(): [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self.force_reraise() [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] raise self.value [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] updated_port = self._update_port( [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] _ensure_no_port_binding_failure(port) [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 833.899953] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] raise exception.PortBindingFailed(port_id=port['id']) [ 833.900291] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] nova.exception.PortBindingFailed: Binding failed for port 075296e6-0c1e-4d55-a362-2fe91040eefb, please check neutron logs for more information. [ 833.900291] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] [ 833.900291] env[63593]: INFO nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Terminating instance [ 833.900814] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Acquiring lock "refresh_cache-94486acf-7f7b-4187-b9d1-332f70488bd5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 833.932454] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 834.114368] env[63593]: DEBUG oslo_concurrency.lockutils [None req-04226f3f-5274-4c29-bbb8-eb0178d28bbd tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Acquiring lock "257c613c-8613-42e5-b400-b9c352a151f0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 834.114601] env[63593]: DEBUG oslo_concurrency.lockutils [None req-04226f3f-5274-4c29-bbb8-eb0178d28bbd tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Lock "257c613c-8613-42e5-b400-b9c352a151f0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 834.195026] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 43b2aa92-913e-4bae-9e25-82cf423d4148 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 834.260789] env[63593]: DEBUG nova.network.neutron [req-fe688d54-8f0a-4c71-ad8b-5399ed306894 req-1fe4d861-8e2a-49de-a8ae-bff1b482c72d service nova] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 834.333910] env[63593]: DEBUG nova.network.neutron [req-fe688d54-8f0a-4c71-ad8b-5399ed306894 req-1fe4d861-8e2a-49de-a8ae-bff1b482c72d service nova] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 834.697862] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance be01dd8d-53fb-4fcc-9bf8-41934b4d0263 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 834.837607] env[63593]: DEBUG oslo_concurrency.lockutils [req-fe688d54-8f0a-4c71-ad8b-5399ed306894 req-1fe4d861-8e2a-49de-a8ae-bff1b482c72d service nova] Releasing lock "refresh_cache-94486acf-7f7b-4187-b9d1-332f70488bd5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 834.837744] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Acquired lock "refresh_cache-94486acf-7f7b-4187-b9d1-332f70488bd5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 834.837945] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 835.202936] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2b478998-632e-4286-8d97-eab4f025bdda has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 835.357233] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 835.436457] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 835.705464] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 464b651f-3f3f-425d-9da6-546ea9610515 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 835.764099] env[63593]: DEBUG nova.compute.manager [req-ab3f0fc9-3d77-4303-9214-0740f6fd9241 req-e25a1069-7655-43f0-bf47-73679b06e9ef service nova] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Received event network-vif-deleted-075296e6-0c1e-4d55-a362-2fe91040eefb {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 835.939965] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Releasing lock "refresh_cache-94486acf-7f7b-4187-b9d1-332f70488bd5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 835.940177] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 835.940378] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 835.940676] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-79720996-76fa-4b8b-bb84-9e822d130f9a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.949831] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21401845-057d-4d80-b932-535e0a0fdb6a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.970137] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 94486acf-7f7b-4187-b9d1-332f70488bd5 could not be found. [ 835.970371] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 835.970555] env[63593]: INFO nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Took 0.03 seconds to destroy the instance on the hypervisor. [ 835.970786] env[63593]: DEBUG oslo.service.loopingcall [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 835.970992] env[63593]: DEBUG nova.compute.manager [-] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 835.971114] env[63593]: DEBUG nova.network.neutron [-] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 835.986627] env[63593]: DEBUG nova.network.neutron [-] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 836.209751] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 9385f942-728d-4041-bc52-77e3c9752b7f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 836.488738] env[63593]: DEBUG nova.network.neutron [-] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 836.712942] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 836.991601] env[63593]: INFO nova.compute.manager [-] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Took 1.02 seconds to deallocate network for instance. [ 836.994227] env[63593]: DEBUG nova.compute.claims [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 836.994419] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.216819] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 991481e2-557e-4222-adfc-d298fa9314a0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.719741] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance ef8646c6-7ead-4c9f-8e8f-85c758a35006 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.222834] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a223dd01-d07d-4ac5-acfc-2e03b4c7dff4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.725846] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d3a4902f-1829-427c-9f37-6e83ae7265e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 839.229133] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 839.733286] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fbd620a6-3743-4737-98b7-b13928b6587d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 840.236646] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 840.740467] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 328b91c6-4ae4-442f-ac87-519658a812ff has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 841.243739] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 841.746928] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 842.250210] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 842.753826] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 843.256834] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d760b552-e2ff-46db-88eb-f5f58fc88239 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 843.257071] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 2 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 843.257222] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=896MB phys_disk=200GB used_disk=2GB total_vcpus=48 used_vcpus=2 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 843.510059] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-136302c9-1622-4eba-b54d-0394d6f2f0ad {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.517568] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd05cf2b-030f-4d21-8412-e0d119eca4b9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.548601] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2b59e37-3731-42ba-ab10-c18d9730e875 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.555014] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d75ee1fd-247d-4579-985f-97368fc90cf8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.567468] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 844.070772] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 844.575680] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 844.576017] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 12.931s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 844.576434] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.358s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.577888] env[63593]: INFO nova.compute.claims [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 845.012669] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 845.012865] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 845.013020] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 845.013189] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 845.013313] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11226}} [ 845.516677] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] There are 0 instances to clean {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11235}} [ 845.516961] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 845.517087] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances with incomplete migration {{(pid=63593) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11264}} [ 845.873486] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1514dd8-3137-47d2-8d98-fd9b621ddf7a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.881138] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd369f4f-e730-4e3f-88e5-6849ec03ecd9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.910498] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d47d9b5-db7c-411a-9515-2c7056122045 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.917247] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5229216b-c7ba-4396-a69c-4c515048b095 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.929726] env[63593]: DEBUG nova.compute.provider_tree [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 846.019316] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 846.432778] env[63593]: DEBUG nova.scheduler.client.report [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 846.937721] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.361s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.938297] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 846.940837] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.842s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.942191] env[63593]: INFO nova.compute.claims [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 847.449303] env[63593]: DEBUG nova.compute.utils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 847.450865] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Not allocating networking since 'none' was specified. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 847.521077] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 847.521320] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 847.521469] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 847.521588] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 847.955917] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 848.024770] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 848.024937] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 848.025091] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 848.025221] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 848.025342] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 848.027795] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.028202] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.028359] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.028537] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.208534] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5288c6e6-01ba-4dea-9a84-4bbcff237a84 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.216538] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92afde17-3520-447a-aaa4-71e34e6ea96e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.247172] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d17925c2-31b2-4081-a4cc-f86ecab43276 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.255063] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73d602da-4fbc-491c-8235-cacc1454e8c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.268436] env[63593]: DEBUG nova.compute.provider_tree [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 848.771195] env[63593]: DEBUG nova.scheduler.client.report [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 848.963878] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 848.989728] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 848.989978] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 848.990181] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 848.990489] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 848.990712] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 848.990869] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 848.991091] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 848.991254] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 848.991419] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 848.991579] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 848.991743] env[63593]: DEBUG nova.virt.hardware [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 848.992589] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca390f7d-a569-4969-a61b-95b0a58837c7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.000501] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26425529-110d-452c-887e-ce6e0c1b9667 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.013829] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 849.014988] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Instance VIF info [] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 849.021281] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Creating folder: Project (86cc5f57b1d040bf811cad826bf2aea5). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 849.021411] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-8b5e6cc1-3f76-4672-a55c-9a94d2b35b68 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.033236] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Created folder: Project (86cc5f57b1d040bf811cad826bf2aea5) in parent group-v291016. [ 849.033411] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Creating folder: Instances. Parent ref: group-v291031. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 849.033614] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-948ba805-5877-419c-a5b2-0db7575512c0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.042762] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Created folder: Instances in parent group-v291031. [ 849.042897] env[63593]: DEBUG oslo.service.loopingcall [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 849.043941] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 849.043941] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-96919683-bdd0-4ff0-bfc9-df936120762a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.058617] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 849.058617] env[63593]: value = "task-1367963" [ 849.058617] env[63593]: _type = "Task" [ 849.058617] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 849.066694] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367963, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 849.275714] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.335s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 849.276326] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 849.278869] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 27.282s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 849.280279] env[63593]: INFO nova.compute.claims [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 849.522857] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 849.568501] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367963, 'name': CreateVM_Task, 'duration_secs': 0.242447} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 849.568501] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 849.568893] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 849.569128] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 849.569471] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 849.569711] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c832e8bf-5a57-4828-89c5-54b655108874 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.574512] env[63593]: DEBUG oslo_vmware.api [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for the task: (returnval){ [ 849.574512] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5272ba00-aa11-120d-d09d-fe23b3e3ce7a" [ 849.574512] env[63593]: _type = "Task" [ 849.574512] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 849.581792] env[63593]: DEBUG oslo_vmware.api [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5272ba00-aa11-120d-d09d-fe23b3e3ce7a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 849.787446] env[63593]: DEBUG nova.compute.utils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 849.788584] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Not allocating networking since 'none' was specified. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 850.084919] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 850.085253] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 850.085421] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 850.085565] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 850.085737] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 850.086042] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-be1dd191-4419-479c-8a19-3ab490f60c1d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.103337] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 850.103539] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 850.104279] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-868b8caf-7c32-47bb-a4f2-14a58339753f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.109247] env[63593]: DEBUG oslo_vmware.api [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for the task: (returnval){ [ 850.109247] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]525086e2-e074-06d3-3de5-b916795ab09f" [ 850.109247] env[63593]: _type = "Task" [ 850.109247] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 850.116282] env[63593]: DEBUG oslo_vmware.api [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]525086e2-e074-06d3-3de5-b916795ab09f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 850.293816] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 850.536565] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b525004-8790-45e6-b09c-d1a6f6cba015 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.544061] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbec0297-c072-4ba1-bb8f-b61beb5344cc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.573315] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97611e35-1c22-4c3f-baec-1c49f6363c35 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.579847] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38209064-24f9-426e-912a-8ed5a8a4d245 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.592308] env[63593]: DEBUG nova.compute.provider_tree [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 850.618984] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 850.619195] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Creating directory with path [datastore2] vmware_temp/bba2d177-d1ca-4c03-82b0-c47a80fb4f61/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 850.619400] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-de3922dd-f883-4386-a40b-1b6c0a096304 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.638388] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Created directory with path [datastore2] vmware_temp/bba2d177-d1ca-4c03-82b0-c47a80fb4f61/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 850.638572] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Fetch image to [datastore2] vmware_temp/bba2d177-d1ca-4c03-82b0-c47a80fb4f61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 850.638735] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/bba2d177-d1ca-4c03-82b0-c47a80fb4f61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 850.639415] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f80a7663-eae1-4334-8c85-af44481bca5f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.645717] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5870e07-79a5-4122-82ee-28b35e58c8ac {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.654164] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85113141-6457-4d6d-8d6b-e94e9b223be4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.683326] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07a23ff9-4453-4085-b049-fd8e0bc54a95 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.689130] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-516404b7-8afa-4ba8-89a8-30e7460bc027 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.714129] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 850.761504] env[63593]: DEBUG oslo_vmware.rw_handles [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/bba2d177-d1ca-4c03-82b0-c47a80fb4f61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 850.826043] env[63593]: DEBUG oslo_vmware.rw_handles [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 850.826254] env[63593]: DEBUG oslo_vmware.rw_handles [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/bba2d177-d1ca-4c03-82b0-c47a80fb4f61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 851.095316] env[63593]: DEBUG nova.scheduler.client.report [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 851.326525] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 851.346321] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 851.347318] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 851.347318] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 851.347318] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 851.347318] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 851.347318] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 851.347543] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 851.347684] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 851.347963] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 851.348258] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 851.348465] env[63593]: DEBUG nova.virt.hardware [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 851.349350] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-248eaacf-d15a-4d9c-95ca-0e361257a980 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.357723] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd60f1f0-cda5-40c6-b2b9-99470a864a66 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.370935] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Instance VIF info [] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 851.376164] env[63593]: DEBUG oslo.service.loopingcall [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 851.376358] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 851.376552] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-6faa4b3c-a2f8-4d7b-b271-d7adac0389f9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.393320] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 851.393320] env[63593]: value = "task-1367964" [ 851.393320] env[63593]: _type = "Task" [ 851.393320] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 851.400993] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367964, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 851.600934] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.322s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 851.601099] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 851.603717] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 27.756s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 851.605131] env[63593]: INFO nova.compute.claims [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 851.904404] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367964, 'name': CreateVM_Task, 'duration_secs': 0.239438} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 851.904582] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 851.904985] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 851.905155] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 851.905477] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 851.905714] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-653b91eb-f8bc-4ae6-a9d2-502c0e7ba90c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.910431] env[63593]: DEBUG oslo_vmware.api [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for the task: (returnval){ [ 851.910431] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]521d8751-ff16-7c91-8fc0-65049ec387e6" [ 851.910431] env[63593]: _type = "Task" [ 851.910431] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 851.917805] env[63593]: DEBUG oslo_vmware.api [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]521d8751-ff16-7c91-8fc0-65049ec387e6, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 852.110080] env[63593]: DEBUG nova.compute.utils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 852.113529] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 852.113692] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 852.165285] env[63593]: DEBUG nova.policy [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b5f8690846194a168c9cf1163a6a946c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '866ad99a435b49b99465a38d47ddf4f9', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 852.421053] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 852.421053] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 852.421053] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 852.468193] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Successfully created port: 3b25f8af-5d71-4ca2-80fe-c544d5569384 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 852.615269] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 852.963711] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-710fdfb2-c826-4100-8a19-b36117dc3516 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.971568] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e313785a-0c15-4787-ac96-0617fdc582d3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.003063] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7843edf-f580-4906-bdb0-aa088c859f27 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.010131] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdd1d5da-3a5a-4978-bf26-f1ce96bfc40e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.023175] env[63593]: DEBUG nova.compute.provider_tree [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 853.180510] env[63593]: DEBUG nova.compute.manager [req-39d9d888-3a86-4a2d-98a7-c3d14a4de0e6 req-c4928729-d3f6-4181-a28f-29265555ac09 service nova] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Received event network-changed-3b25f8af-5d71-4ca2-80fe-c544d5569384 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 853.180762] env[63593]: DEBUG nova.compute.manager [req-39d9d888-3a86-4a2d-98a7-c3d14a4de0e6 req-c4928729-d3f6-4181-a28f-29265555ac09 service nova] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Refreshing instance network info cache due to event network-changed-3b25f8af-5d71-4ca2-80fe-c544d5569384. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 853.181012] env[63593]: DEBUG oslo_concurrency.lockutils [req-39d9d888-3a86-4a2d-98a7-c3d14a4de0e6 req-c4928729-d3f6-4181-a28f-29265555ac09 service nova] Acquiring lock "refresh_cache-43b2aa92-913e-4bae-9e25-82cf423d4148" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 853.181096] env[63593]: DEBUG oslo_concurrency.lockutils [req-39d9d888-3a86-4a2d-98a7-c3d14a4de0e6 req-c4928729-d3f6-4181-a28f-29265555ac09 service nova] Acquired lock "refresh_cache-43b2aa92-913e-4bae-9e25-82cf423d4148" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 853.181259] env[63593]: DEBUG nova.network.neutron [req-39d9d888-3a86-4a2d-98a7-c3d14a4de0e6 req-c4928729-d3f6-4181-a28f-29265555ac09 service nova] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Refreshing network info cache for port 3b25f8af-5d71-4ca2-80fe-c544d5569384 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 853.368866] env[63593]: ERROR nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3b25f8af-5d71-4ca2-80fe-c544d5569384, please check neutron logs for more information. [ 853.368866] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 853.368866] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 853.368866] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 853.368866] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 853.368866] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 853.368866] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 853.368866] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 853.368866] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 853.368866] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 853.368866] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 853.368866] env[63593]: ERROR nova.compute.manager raise self.value [ 853.368866] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 853.368866] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 853.368866] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 853.368866] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 853.369461] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 853.369461] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 853.369461] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3b25f8af-5d71-4ca2-80fe-c544d5569384, please check neutron logs for more information. [ 853.369461] env[63593]: ERROR nova.compute.manager [ 853.369461] env[63593]: Traceback (most recent call last): [ 853.369461] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 853.369461] env[63593]: listener.cb(fileno) [ 853.369461] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 853.369461] env[63593]: result = function(*args, **kwargs) [ 853.369461] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 853.369461] env[63593]: return func(*args, **kwargs) [ 853.369461] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 853.369461] env[63593]: raise e [ 853.369461] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 853.369461] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 853.369461] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 853.369461] env[63593]: created_port_ids = self._update_ports_for_instance( [ 853.369461] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 853.369461] env[63593]: with excutils.save_and_reraise_exception(): [ 853.369461] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 853.369461] env[63593]: self.force_reraise() [ 853.369461] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 853.369461] env[63593]: raise self.value [ 853.369461] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 853.369461] env[63593]: updated_port = self._update_port( [ 853.369461] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 853.369461] env[63593]: _ensure_no_port_binding_failure(port) [ 853.369461] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 853.369461] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 853.370376] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 3b25f8af-5d71-4ca2-80fe-c544d5569384, please check neutron logs for more information. [ 853.370376] env[63593]: Removing descriptor: 18 [ 853.526551] env[63593]: DEBUG nova.scheduler.client.report [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 853.630654] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 853.654692] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 853.654925] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 853.655089] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 853.655271] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 853.655413] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 853.655552] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 853.655747] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 853.655899] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 853.656088] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 853.656269] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 853.656438] env[63593]: DEBUG nova.virt.hardware [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 853.657300] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adc8d901-8296-40df-95b4-405d8279fbd1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.665033] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f531a1be-23f6-4962-89c9-80ef91ef054a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.678596] env[63593]: ERROR nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3b25f8af-5d71-4ca2-80fe-c544d5569384, please check neutron logs for more information. [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Traceback (most recent call last): [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] yield resources [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self.driver.spawn(context, instance, image_meta, [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self._vmops.spawn(context, instance, image_meta, injected_files, [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] vm_ref = self.build_virtual_machine(instance, [ 853.678596] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] vif_infos = vmwarevif.get_vif_info(self._session, [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] for vif in network_info: [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] return self._sync_wrapper(fn, *args, **kwargs) [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self.wait() [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self[:] = self._gt.wait() [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] return self._exit_event.wait() [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 853.678988] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] current.throw(*self._exc) [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] result = function(*args, **kwargs) [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] return func(*args, **kwargs) [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] raise e [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] nwinfo = self.network_api.allocate_for_instance( [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] created_port_ids = self._update_ports_for_instance( [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] with excutils.save_and_reraise_exception(): [ 853.679418] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self.force_reraise() [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] raise self.value [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] updated_port = self._update_port( [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] _ensure_no_port_binding_failure(port) [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] raise exception.PortBindingFailed(port_id=port['id']) [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] nova.exception.PortBindingFailed: Binding failed for port 3b25f8af-5d71-4ca2-80fe-c544d5569384, please check neutron logs for more information. [ 853.679827] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] [ 853.679827] env[63593]: INFO nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Terminating instance [ 853.681430] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Acquiring lock "refresh_cache-43b2aa92-913e-4bae-9e25-82cf423d4148" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 853.697884] env[63593]: DEBUG nova.network.neutron [req-39d9d888-3a86-4a2d-98a7-c3d14a4de0e6 req-c4928729-d3f6-4181-a28f-29265555ac09 service nova] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 853.772609] env[63593]: DEBUG nova.network.neutron [req-39d9d888-3a86-4a2d-98a7-c3d14a4de0e6 req-c4928729-d3f6-4181-a28f-29265555ac09 service nova] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 854.031497] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.428s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 854.032683] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 28.350s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 854.036200] env[63593]: INFO nova.compute.claims [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 854.275374] env[63593]: DEBUG oslo_concurrency.lockutils [req-39d9d888-3a86-4a2d-98a7-c3d14a4de0e6 req-c4928729-d3f6-4181-a28f-29265555ac09 service nova] Releasing lock "refresh_cache-43b2aa92-913e-4bae-9e25-82cf423d4148" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 854.275796] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Acquired lock "refresh_cache-43b2aa92-913e-4bae-9e25-82cf423d4148" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 854.275982] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 854.540870] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Acquiring lock "121e44bc-2bc5-4fa2-bf28-994ba4ec87e9" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 854.541194] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Lock "121e44bc-2bc5-4fa2-bf28-994ba4ec87e9" acquired by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 854.793206] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 854.861714] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 855.046117] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Lock "121e44bc-2bc5-4fa2-bf28-994ba4ec87e9" "released" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: held 0.504s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 855.046232] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 855.204174] env[63593]: DEBUG nova.compute.manager [req-fb7a5879-8976-4e38-9119-038511c6da56 req-ee6bc465-5673-4c64-a947-99cd4724da97 service nova] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Received event network-vif-deleted-3b25f8af-5d71-4ca2-80fe-c544d5569384 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 855.306865] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a27a441c-ce7b-493b-8909-e5cc390e4e03 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.314686] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ed53496-931d-4ef8-943f-82a6ff84c304 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.344044] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e72b386-2bd4-4332-b0da-de013d7eedbe {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.351701] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6bab75ce-be38-4e81-9cf2-7314f8a8d07c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.366858] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Releasing lock "refresh_cache-43b2aa92-913e-4bae-9e25-82cf423d4148" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 855.367285] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 855.367476] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 855.367926] env[63593]: DEBUG nova.compute.provider_tree [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 855.369088] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0173d500-3323-4029-80fc-8ae41f02bd8b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.377008] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-222135cd-8597-4ffa-a4f4-ed4e62564890 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.400314] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 43b2aa92-913e-4bae-9e25-82cf423d4148 could not be found. [ 855.400506] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 855.400680] env[63593]: INFO nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Took 0.03 seconds to destroy the instance on the hypervisor. [ 855.400905] env[63593]: DEBUG oslo.service.loopingcall [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 855.401163] env[63593]: DEBUG nova.compute.manager [-] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 855.401262] env[63593]: DEBUG nova.network.neutron [-] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 855.415346] env[63593]: DEBUG nova.network.neutron [-] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 855.554087] env[63593]: DEBUG nova.compute.utils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 855.555792] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 855.555998] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 855.605870] env[63593]: DEBUG nova.policy [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '46e84012846a49519df24a45b998499f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '215432c19c184e3f94366039907d3044', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 855.866130] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Successfully created port: 24aa41a9-9a5b-48d3-80b9-2b5cf03df989 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 855.871958] env[63593]: DEBUG nova.scheduler.client.report [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 855.919592] env[63593]: DEBUG nova.network.neutron [-] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 856.058722] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 856.378233] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.344s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 856.378233] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 856.381046] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 27.976s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 856.421372] env[63593]: INFO nova.compute.manager [-] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Took 1.02 seconds to deallocate network for instance. [ 856.425181] env[63593]: DEBUG nova.compute.claims [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 856.425381] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 856.810245] env[63593]: ERROR nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989, please check neutron logs for more information. [ 856.810245] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 856.810245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 856.810245] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 856.810245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 856.810245] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 856.810245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 856.810245] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 856.810245] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 856.810245] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 856.810245] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 856.810245] env[63593]: ERROR nova.compute.manager raise self.value [ 856.810245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 856.810245] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 856.810245] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 856.810245] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 856.810863] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 856.810863] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 856.810863] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989, please check neutron logs for more information. [ 856.810863] env[63593]: ERROR nova.compute.manager [ 856.810863] env[63593]: Traceback (most recent call last): [ 856.810863] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 856.810863] env[63593]: listener.cb(fileno) [ 856.810863] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 856.810863] env[63593]: result = function(*args, **kwargs) [ 856.810863] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 856.810863] env[63593]: return func(*args, **kwargs) [ 856.810863] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 856.810863] env[63593]: raise e [ 856.810863] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 856.810863] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 856.810863] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 856.810863] env[63593]: created_port_ids = self._update_ports_for_instance( [ 856.810863] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 856.810863] env[63593]: with excutils.save_and_reraise_exception(): [ 856.810863] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 856.810863] env[63593]: self.force_reraise() [ 856.810863] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 856.810863] env[63593]: raise self.value [ 856.810863] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 856.810863] env[63593]: updated_port = self._update_port( [ 856.810863] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 856.810863] env[63593]: _ensure_no_port_binding_failure(port) [ 856.810863] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 856.810863] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 856.811905] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989, please check neutron logs for more information. [ 856.811905] env[63593]: Removing descriptor: 18 [ 856.885495] env[63593]: DEBUG nova.compute.utils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 856.886888] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 856.887074] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 856.927108] env[63593]: DEBUG nova.policy [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3bb930b4d1534f3ea3abde038aa60100', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3101ac6769714328b1b7e2c5f371bccc', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 857.025231] env[63593]: DEBUG nova.scheduler.client.report [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Refreshing inventories for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 857.047516] env[63593]: DEBUG nova.scheduler.client.report [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Updating ProviderTree inventory for provider 7345f17f-3af2-4b0a-9521-0292dc691877 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 857.047673] env[63593]: DEBUG nova.compute.provider_tree [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Updating inventory in ProviderTree for provider 7345f17f-3af2-4b0a-9521-0292dc691877 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 857.060247] env[63593]: DEBUG nova.scheduler.client.report [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Refreshing aggregate associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, aggregates: None {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 857.071675] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 857.080792] env[63593]: DEBUG nova.scheduler.client.report [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Refreshing trait associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 857.109068] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 857.109068] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 857.109248] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 857.109325] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 857.109460] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 857.109598] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 857.109799] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 857.109947] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 857.110118] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 857.110734] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 857.110734] env[63593]: DEBUG nova.virt.hardware [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 857.111400] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d2575a0-4bdd-4bab-b747-fa31d97b3209 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.120851] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4be6cb79-0143-407a-8a34-d1830d8a0810 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.139593] env[63593]: ERROR nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989, please check neutron logs for more information. [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Traceback (most recent call last): [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] yield resources [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self.driver.spawn(context, instance, image_meta, [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self._vmops.spawn(context, instance, image_meta, injected_files, [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] vm_ref = self.build_virtual_machine(instance, [ 857.139593] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] vif_infos = vmwarevif.get_vif_info(self._session, [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] for vif in network_info: [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] return self._sync_wrapper(fn, *args, **kwargs) [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self.wait() [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self[:] = self._gt.wait() [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] return self._exit_event.wait() [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 857.139997] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] current.throw(*self._exc) [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] result = function(*args, **kwargs) [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] return func(*args, **kwargs) [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] raise e [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] nwinfo = self.network_api.allocate_for_instance( [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] created_port_ids = self._update_ports_for_instance( [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] with excutils.save_and_reraise_exception(): [ 857.140391] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self.force_reraise() [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] raise self.value [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] updated_port = self._update_port( [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] _ensure_no_port_binding_failure(port) [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] raise exception.PortBindingFailed(port_id=port['id']) [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] nova.exception.PortBindingFailed: Binding failed for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989, please check neutron logs for more information. [ 857.140931] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] [ 857.140931] env[63593]: INFO nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Terminating instance [ 857.145630] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Acquiring lock "refresh_cache-be01dd8d-53fb-4fcc-9bf8-41934b4d0263" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 857.145796] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Acquired lock "refresh_cache-be01dd8d-53fb-4fcc-9bf8-41934b4d0263" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 857.145963] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 857.226410] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Successfully created port: a573cab4-883b-4744-b3c9-c6590951a92c {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 857.232891] env[63593]: DEBUG nova.compute.manager [req-c7979ea1-8974-4944-9833-42ee49920195 req-b1114972-e2a2-4d75-93d2-f2446a86c33c service nova] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Received event network-changed-24aa41a9-9a5b-48d3-80b9-2b5cf03df989 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 857.236153] env[63593]: DEBUG nova.compute.manager [req-c7979ea1-8974-4944-9833-42ee49920195 req-b1114972-e2a2-4d75-93d2-f2446a86c33c service nova] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Refreshing instance network info cache due to event network-changed-24aa41a9-9a5b-48d3-80b9-2b5cf03df989. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 857.236153] env[63593]: DEBUG oslo_concurrency.lockutils [req-c7979ea1-8974-4944-9833-42ee49920195 req-b1114972-e2a2-4d75-93d2-f2446a86c33c service nova] Acquiring lock "refresh_cache-be01dd8d-53fb-4fcc-9bf8-41934b4d0263" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 857.392887] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 857.434336] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c06d07d5-070b-4efb-9d9e-e80475bc20a8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.442922] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9efe4f9e-2d03-4d71-9ea9-a6845b30bdd6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.475135] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b9a24e8-2a5c-47e8-ad41-d0a45c119d5b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.482992] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebecc399-ea63-4234-b768-50de95159c6d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.496925] env[63593]: DEBUG nova.compute.provider_tree [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 857.667653] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 857.750960] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 858.003939] env[63593]: DEBUG nova.scheduler.client.report [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 858.253666] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Releasing lock "refresh_cache-be01dd8d-53fb-4fcc-9bf8-41934b4d0263" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 858.254113] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 858.254308] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 858.254906] env[63593]: DEBUG oslo_concurrency.lockutils [req-c7979ea1-8974-4944-9833-42ee49920195 req-b1114972-e2a2-4d75-93d2-f2446a86c33c service nova] Acquired lock "refresh_cache-be01dd8d-53fb-4fcc-9bf8-41934b4d0263" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 858.255556] env[63593]: DEBUG nova.network.neutron [req-c7979ea1-8974-4944-9833-42ee49920195 req-b1114972-e2a2-4d75-93d2-f2446a86c33c service nova] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Refreshing network info cache for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 858.256022] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6aefb5c9-60b2-4215-a12c-bfa9df3ae5f0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.264637] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5046078-2772-4d23-939c-0e9712a3e3df {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.287927] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance be01dd8d-53fb-4fcc-9bf8-41934b4d0263 could not be found. [ 858.287927] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 858.288034] env[63593]: INFO nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Took 0.03 seconds to destroy the instance on the hypervisor. [ 858.289021] env[63593]: DEBUG oslo.service.loopingcall [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 858.289021] env[63593]: DEBUG nova.compute.manager [-] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 858.289021] env[63593]: DEBUG nova.network.neutron [-] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 858.303325] env[63593]: DEBUG nova.network.neutron [-] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 858.325523] env[63593]: ERROR nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a573cab4-883b-4744-b3c9-c6590951a92c, please check neutron logs for more information. [ 858.325523] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 858.325523] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 858.325523] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 858.325523] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 858.325523] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 858.325523] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 858.325523] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 858.325523] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 858.325523] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 858.325523] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 858.325523] env[63593]: ERROR nova.compute.manager raise self.value [ 858.325523] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 858.325523] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 858.325523] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 858.325523] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 858.326129] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 858.326129] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 858.326129] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a573cab4-883b-4744-b3c9-c6590951a92c, please check neutron logs for more information. [ 858.326129] env[63593]: ERROR nova.compute.manager [ 858.326129] env[63593]: Traceback (most recent call last): [ 858.326129] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 858.326129] env[63593]: listener.cb(fileno) [ 858.326129] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 858.326129] env[63593]: result = function(*args, **kwargs) [ 858.326129] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 858.326129] env[63593]: return func(*args, **kwargs) [ 858.326129] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 858.326129] env[63593]: raise e [ 858.326129] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 858.326129] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 858.326129] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 858.326129] env[63593]: created_port_ids = self._update_ports_for_instance( [ 858.326129] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 858.326129] env[63593]: with excutils.save_and_reraise_exception(): [ 858.326129] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 858.326129] env[63593]: self.force_reraise() [ 858.326129] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 858.326129] env[63593]: raise self.value [ 858.326129] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 858.326129] env[63593]: updated_port = self._update_port( [ 858.326129] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 858.326129] env[63593]: _ensure_no_port_binding_failure(port) [ 858.326129] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 858.326129] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 858.327160] env[63593]: nova.exception.PortBindingFailed: Binding failed for port a573cab4-883b-4744-b3c9-c6590951a92c, please check neutron logs for more information. [ 858.327160] env[63593]: Removing descriptor: 18 [ 858.403063] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 858.427170] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 858.427413] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 858.427567] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 858.427741] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 858.427885] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 858.428038] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 858.428272] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 858.428432] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 858.428594] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 858.428748] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 858.428908] env[63593]: DEBUG nova.virt.hardware [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 858.429774] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9683560-9b2d-4989-8af4-9ac5987e1161 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.437355] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a82d63de-9a95-4af1-9b33-49f57eba0600 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 858.451081] env[63593]: ERROR nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a573cab4-883b-4744-b3c9-c6590951a92c, please check neutron logs for more information. [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Traceback (most recent call last): [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] yield resources [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self.driver.spawn(context, instance, image_meta, [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self._vmops.spawn(context, instance, image_meta, injected_files, [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] vm_ref = self.build_virtual_machine(instance, [ 858.451081] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] vif_infos = vmwarevif.get_vif_info(self._session, [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] for vif in network_info: [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] return self._sync_wrapper(fn, *args, **kwargs) [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self.wait() [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self[:] = self._gt.wait() [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] return self._exit_event.wait() [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 858.451540] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] current.throw(*self._exc) [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] result = function(*args, **kwargs) [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] return func(*args, **kwargs) [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] raise e [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] nwinfo = self.network_api.allocate_for_instance( [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] created_port_ids = self._update_ports_for_instance( [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] with excutils.save_and_reraise_exception(): [ 858.451962] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self.force_reraise() [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] raise self.value [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] updated_port = self._update_port( [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] _ensure_no_port_binding_failure(port) [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] raise exception.PortBindingFailed(port_id=port['id']) [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] nova.exception.PortBindingFailed: Binding failed for port a573cab4-883b-4744-b3c9-c6590951a92c, please check neutron logs for more information. [ 858.452402] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] [ 858.452402] env[63593]: INFO nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Terminating instance [ 858.453344] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "refresh_cache-2b478998-632e-4286-8d97-eab4f025bdda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 858.453501] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "refresh_cache-2b478998-632e-4286-8d97-eab4f025bdda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 858.453662] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 858.508745] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.128s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 858.509414] env[63593]: ERROR nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6a72110e-d211-42af-9cab-dc99d21308f2, please check neutron logs for more information. [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Traceback (most recent call last): [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self.driver.spawn(context, instance, image_meta, [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] vm_ref = self.build_virtual_machine(instance, [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] vif_infos = vmwarevif.get_vif_info(self._session, [ 858.509414] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] for vif in network_info: [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] return self._sync_wrapper(fn, *args, **kwargs) [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self.wait() [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self[:] = self._gt.wait() [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] return self._exit_event.wait() [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] result = hub.switch() [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 858.509831] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] return self.greenlet.switch() [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] result = function(*args, **kwargs) [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] return func(*args, **kwargs) [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] raise e [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] nwinfo = self.network_api.allocate_for_instance( [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] created_port_ids = self._update_ports_for_instance( [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] with excutils.save_and_reraise_exception(): [ 858.510260] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] self.force_reraise() [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] raise self.value [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] updated_port = self._update_port( [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] _ensure_no_port_binding_failure(port) [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] raise exception.PortBindingFailed(port_id=port['id']) [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] nova.exception.PortBindingFailed: Binding failed for port 6a72110e-d211-42af-9cab-dc99d21308f2, please check neutron logs for more information. [ 858.510684] env[63593]: ERROR nova.compute.manager [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] [ 858.511039] env[63593]: DEBUG nova.compute.utils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Binding failed for port 6a72110e-d211-42af-9cab-dc99d21308f2, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 858.511215] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 28.439s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 858.512711] env[63593]: INFO nova.compute.claims [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 858.515959] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Build of instance 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a was re-scheduled: Binding failed for port 6a72110e-d211-42af-9cab-dc99d21308f2, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 858.516279] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 858.516563] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Acquiring lock "refresh_cache-9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 858.516730] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Acquired lock "refresh_cache-9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 858.516924] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 858.779935] env[63593]: DEBUG nova.network.neutron [req-c7979ea1-8974-4944-9833-42ee49920195 req-b1114972-e2a2-4d75-93d2-f2446a86c33c service nova] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 858.805494] env[63593]: DEBUG nova.network.neutron [-] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 858.867401] env[63593]: DEBUG nova.network.neutron [req-c7979ea1-8974-4944-9833-42ee49920195 req-b1114972-e2a2-4d75-93d2-f2446a86c33c service nova] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 858.969071] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 859.035647] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 859.037216] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 859.081099] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 859.254762] env[63593]: DEBUG nova.compute.manager [req-c3ddc031-6e7d-49da-8054-ddd44494ef2a req-668a5d45-0071-4d3a-bec4-4bd9e5629b91 service nova] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Received event network-changed-a573cab4-883b-4744-b3c9-c6590951a92c {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 859.254959] env[63593]: DEBUG nova.compute.manager [req-c3ddc031-6e7d-49da-8054-ddd44494ef2a req-668a5d45-0071-4d3a-bec4-4bd9e5629b91 service nova] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Refreshing instance network info cache due to event network-changed-a573cab4-883b-4744-b3c9-c6590951a92c. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 859.255170] env[63593]: DEBUG oslo_concurrency.lockutils [req-c3ddc031-6e7d-49da-8054-ddd44494ef2a req-668a5d45-0071-4d3a-bec4-4bd9e5629b91 service nova] Acquiring lock "refresh_cache-2b478998-632e-4286-8d97-eab4f025bdda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 859.308080] env[63593]: INFO nova.compute.manager [-] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Took 1.02 seconds to deallocate network for instance. [ 859.310590] env[63593]: DEBUG nova.compute.claims [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 859.310777] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 859.369836] env[63593]: DEBUG oslo_concurrency.lockutils [req-c7979ea1-8974-4944-9833-42ee49920195 req-b1114972-e2a2-4d75-93d2-f2446a86c33c service nova] Releasing lock "refresh_cache-be01dd8d-53fb-4fcc-9bf8-41934b4d0263" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 859.370114] env[63593]: DEBUG nova.compute.manager [req-c7979ea1-8974-4944-9833-42ee49920195 req-b1114972-e2a2-4d75-93d2-f2446a86c33c service nova] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Received event network-vif-deleted-24aa41a9-9a5b-48d3-80b9-2b5cf03df989 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 859.540069] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "refresh_cache-2b478998-632e-4286-8d97-eab4f025bdda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 859.540499] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 859.540701] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 859.541012] env[63593]: DEBUG oslo_concurrency.lockutils [req-c3ddc031-6e7d-49da-8054-ddd44494ef2a req-668a5d45-0071-4d3a-bec4-4bd9e5629b91 service nova] Acquired lock "refresh_cache-2b478998-632e-4286-8d97-eab4f025bdda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 859.541185] env[63593]: DEBUG nova.network.neutron [req-c3ddc031-6e7d-49da-8054-ddd44494ef2a req-668a5d45-0071-4d3a-bec4-4bd9e5629b91 service nova] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Refreshing network info cache for port a573cab4-883b-4744-b3c9-c6590951a92c {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 859.542212] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2aaca28f-44a2-48f2-9c9b-7eed7193228e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.553397] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae6b3290-6e83-492a-b133-421f6a6249f6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.579100] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2b478998-632e-4286-8d97-eab4f025bdda could not be found. [ 859.579235] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 859.579398] env[63593]: INFO nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Took 0.04 seconds to destroy the instance on the hypervisor. [ 859.579624] env[63593]: DEBUG oslo.service.loopingcall [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 859.579831] env[63593]: DEBUG nova.compute.manager [-] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 859.579922] env[63593]: DEBUG nova.network.neutron [-] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 859.583135] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Releasing lock "refresh_cache-9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 859.583238] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 859.583397] env[63593]: DEBUG nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 859.583557] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 859.596116] env[63593]: DEBUG nova.network.neutron [-] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 859.604024] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 859.794955] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a50393f-3a0c-4477-893b-e1302154adcc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.802514] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-606b8255-8a1a-4aab-a99c-87776b576e15 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.832147] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69d293b8-3a1b-40c5-898b-f53cd60a6779 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.839504] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-240ae373-338a-4a74-b7cb-9bc3dba25da2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.852351] env[63593]: DEBUG nova.compute.provider_tree [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 860.059399] env[63593]: DEBUG nova.network.neutron [req-c3ddc031-6e7d-49da-8054-ddd44494ef2a req-668a5d45-0071-4d3a-bec4-4bd9e5629b91 service nova] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 860.098710] env[63593]: DEBUG nova.network.neutron [-] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 860.106068] env[63593]: DEBUG nova.network.neutron [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 860.126720] env[63593]: DEBUG nova.network.neutron [req-c3ddc031-6e7d-49da-8054-ddd44494ef2a req-668a5d45-0071-4d3a-bec4-4bd9e5629b91 service nova] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 860.355263] env[63593]: DEBUG nova.scheduler.client.report [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 860.601413] env[63593]: INFO nova.compute.manager [-] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Took 1.02 seconds to deallocate network for instance. [ 860.603768] env[63593]: DEBUG nova.compute.claims [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 860.603946] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 860.608543] env[63593]: INFO nova.compute.manager [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] [instance: 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a] Took 1.02 seconds to deallocate network for instance. [ 860.629098] env[63593]: DEBUG oslo_concurrency.lockutils [req-c3ddc031-6e7d-49da-8054-ddd44494ef2a req-668a5d45-0071-4d3a-bec4-4bd9e5629b91 service nova] Releasing lock "refresh_cache-2b478998-632e-4286-8d97-eab4f025bdda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 860.629098] env[63593]: DEBUG nova.compute.manager [req-c3ddc031-6e7d-49da-8054-ddd44494ef2a req-668a5d45-0071-4d3a-bec4-4bd9e5629b91 service nova] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Received event network-vif-deleted-a573cab4-883b-4744-b3c9-c6590951a92c {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 860.859874] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.349s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 860.860501] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 860.864194] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 28.837s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 860.864982] env[63593]: INFO nova.compute.claims [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 861.369898] env[63593]: DEBUG nova.compute.utils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 861.372955] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 861.372993] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 861.410899] env[63593]: DEBUG nova.policy [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f7c5f59c37074e109c69418134c377b0', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd85f5be4c6bb496a8dca1df0f94d7ec5', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 861.636988] env[63593]: INFO nova.scheduler.client.report [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Deleted allocations for instance 9fedad99-8b1e-4eaa-8382-b76f4dc8d72a [ 861.704060] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Successfully created port: 2c459dc7-c902-48f5-a548-588973a6e93b {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 861.876039] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 862.146224] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a52083f7-3bdd-42be-a869-0baea73ceb90 tempest-TenantUsagesTestJSON-837819617 tempest-TenantUsagesTestJSON-837819617-project-member] Lock "9fedad99-8b1e-4eaa-8382-b76f4dc8d72a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 166.558s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 862.164374] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f6cfa53-cafc-403b-b916-fb22ce182c40 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.173240] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3da3f69d-8d25-4f07-acbf-92ccd19deb26 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.203455] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3df344b-997e-426c-961b-710010ed6fef {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.211776] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37ec23bf-f94a-49a3-9d98-a9b807415f94 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.226605] env[63593]: DEBUG nova.compute.provider_tree [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 862.444237] env[63593]: DEBUG nova.compute.manager [req-d83627e8-5bcb-4c49-aacf-8abdff16239b req-c4e03c19-a3c1-4f51-ad4a-084c3ff98e6e service nova] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Received event network-changed-2c459dc7-c902-48f5-a548-588973a6e93b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 862.444514] env[63593]: DEBUG nova.compute.manager [req-d83627e8-5bcb-4c49-aacf-8abdff16239b req-c4e03c19-a3c1-4f51-ad4a-084c3ff98e6e service nova] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Refreshing instance network info cache due to event network-changed-2c459dc7-c902-48f5-a548-588973a6e93b. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 862.445420] env[63593]: DEBUG oslo_concurrency.lockutils [req-d83627e8-5bcb-4c49-aacf-8abdff16239b req-c4e03c19-a3c1-4f51-ad4a-084c3ff98e6e service nova] Acquiring lock "refresh_cache-464b651f-3f3f-425d-9da6-546ea9610515" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 862.445420] env[63593]: DEBUG oslo_concurrency.lockutils [req-d83627e8-5bcb-4c49-aacf-8abdff16239b req-c4e03c19-a3c1-4f51-ad4a-084c3ff98e6e service nova] Acquired lock "refresh_cache-464b651f-3f3f-425d-9da6-546ea9610515" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 862.445420] env[63593]: DEBUG nova.network.neutron [req-d83627e8-5bcb-4c49-aacf-8abdff16239b req-c4e03c19-a3c1-4f51-ad4a-084c3ff98e6e service nova] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Refreshing network info cache for port 2c459dc7-c902-48f5-a548-588973a6e93b {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 862.616270] env[63593]: ERROR nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2c459dc7-c902-48f5-a548-588973a6e93b, please check neutron logs for more information. [ 862.616270] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 862.616270] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 862.616270] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 862.616270] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 862.616270] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 862.616270] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 862.616270] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 862.616270] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 862.616270] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 862.616270] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 862.616270] env[63593]: ERROR nova.compute.manager raise self.value [ 862.616270] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 862.616270] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 862.616270] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 862.616270] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 862.616699] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 862.616699] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 862.616699] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2c459dc7-c902-48f5-a548-588973a6e93b, please check neutron logs for more information. [ 862.616699] env[63593]: ERROR nova.compute.manager [ 862.616699] env[63593]: Traceback (most recent call last): [ 862.616699] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 862.616699] env[63593]: listener.cb(fileno) [ 862.616699] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 862.616699] env[63593]: result = function(*args, **kwargs) [ 862.616699] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 862.616699] env[63593]: return func(*args, **kwargs) [ 862.616699] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 862.616699] env[63593]: raise e [ 862.616699] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 862.616699] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 862.616699] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 862.616699] env[63593]: created_port_ids = self._update_ports_for_instance( [ 862.616699] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 862.616699] env[63593]: with excutils.save_and_reraise_exception(): [ 862.616699] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 862.616699] env[63593]: self.force_reraise() [ 862.616699] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 862.616699] env[63593]: raise self.value [ 862.616699] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 862.616699] env[63593]: updated_port = self._update_port( [ 862.616699] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 862.616699] env[63593]: _ensure_no_port_binding_failure(port) [ 862.616699] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 862.616699] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 862.617406] env[63593]: nova.exception.PortBindingFailed: Binding failed for port 2c459dc7-c902-48f5-a548-588973a6e93b, please check neutron logs for more information. [ 862.617406] env[63593]: Removing descriptor: 18 [ 862.648559] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 862.729483] env[63593]: DEBUG nova.scheduler.client.report [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 862.888276] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 862.912769] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 862.912889] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 862.912929] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 862.913120] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 862.913276] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 862.913416] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 862.913613] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 862.913766] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 862.913927] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 862.914097] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 862.914269] env[63593]: DEBUG nova.virt.hardware [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 862.915124] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49f3794e-3b90-44ac-9904-4846e3ececc0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.924526] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ebeb6ad-b0a9-4677-9416-8baad0692eef {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.938007] env[63593]: ERROR nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2c459dc7-c902-48f5-a548-588973a6e93b, please check neutron logs for more information. [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Traceback (most recent call last): [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] yield resources [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self.driver.spawn(context, instance, image_meta, [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self._vmops.spawn(context, instance, image_meta, injected_files, [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] vm_ref = self.build_virtual_machine(instance, [ 862.938007] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] vif_infos = vmwarevif.get_vif_info(self._session, [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] for vif in network_info: [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] return self._sync_wrapper(fn, *args, **kwargs) [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self.wait() [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self[:] = self._gt.wait() [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] return self._exit_event.wait() [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 862.938368] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] current.throw(*self._exc) [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] result = function(*args, **kwargs) [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] return func(*args, **kwargs) [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] raise e [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] nwinfo = self.network_api.allocate_for_instance( [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] created_port_ids = self._update_ports_for_instance( [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] with excutils.save_and_reraise_exception(): [ 862.938682] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self.force_reraise() [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] raise self.value [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] updated_port = self._update_port( [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] _ensure_no_port_binding_failure(port) [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] raise exception.PortBindingFailed(port_id=port['id']) [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] nova.exception.PortBindingFailed: Binding failed for port 2c459dc7-c902-48f5-a548-588973a6e93b, please check neutron logs for more information. [ 862.939048] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] [ 862.939048] env[63593]: INFO nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Terminating instance [ 862.940363] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Acquiring lock "refresh_cache-464b651f-3f3f-425d-9da6-546ea9610515" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 862.966812] env[63593]: DEBUG nova.network.neutron [req-d83627e8-5bcb-4c49-aacf-8abdff16239b req-c4e03c19-a3c1-4f51-ad4a-084c3ff98e6e service nova] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 863.105024] env[63593]: DEBUG nova.network.neutron [req-d83627e8-5bcb-4c49-aacf-8abdff16239b req-c4e03c19-a3c1-4f51-ad4a-084c3ff98e6e service nova] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 863.172120] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 863.234755] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.371s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 863.235350] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 863.238623] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.306s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 863.240088] env[63593]: INFO nova.compute.claims [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 863.608783] env[63593]: DEBUG oslo_concurrency.lockutils [req-d83627e8-5bcb-4c49-aacf-8abdff16239b req-c4e03c19-a3c1-4f51-ad4a-084c3ff98e6e service nova] Releasing lock "refresh_cache-464b651f-3f3f-425d-9da6-546ea9610515" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 863.609222] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Acquired lock "refresh_cache-464b651f-3f3f-425d-9da6-546ea9610515" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 863.609438] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 863.744136] env[63593]: DEBUG nova.compute.utils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 863.748396] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 863.748611] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 863.803069] env[63593]: DEBUG nova.policy [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '33be5e80c65444eaaf2e78b4ee5e497a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cfe14210e5b1405d85ade6e6d022b215', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 864.103221] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Successfully created port: b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 864.130722] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 864.248999] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 864.297370] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 864.485077] env[63593]: DEBUG nova.compute.manager [req-af10fe41-d5a2-469f-b983-c1cb853cb03e req-1b2f61b4-b4fa-44e7-83bd-390eb901ea03 service nova] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Received event network-vif-deleted-2c459dc7-c902-48f5-a548-588973a6e93b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 864.555545] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa806567-f37d-42c4-8afa-cf45793ec88a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.563077] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d785e0d-729f-437e-a156-d91ff129c3ce {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.593703] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-454c87e7-c23a-4315-b185-b5c0b51e927c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.600680] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96284b56-c6fe-4c01-ac73-b93d626eaa2d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.613600] env[63593]: DEBUG nova.compute.provider_tree [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 864.799133] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Releasing lock "refresh_cache-464b651f-3f3f-425d-9da6-546ea9610515" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 864.799535] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 864.799723] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 864.800272] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-94d34c17-359a-46c5-82c6-25dcc3b4324b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.809407] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de4271de-a520-4859-bef2-32a3c8baa594 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.831859] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 464b651f-3f3f-425d-9da6-546ea9610515 could not be found. [ 864.832091] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 864.832272] env[63593]: INFO nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Took 0.03 seconds to destroy the instance on the hypervisor. [ 864.832506] env[63593]: DEBUG oslo.service.loopingcall [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 864.832719] env[63593]: DEBUG nova.compute.manager [-] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 864.832813] env[63593]: DEBUG nova.network.neutron [-] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 864.849299] env[63593]: DEBUG nova.network.neutron [-] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 865.116341] env[63593]: DEBUG nova.scheduler.client.report [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 865.142995] env[63593]: ERROR nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5, please check neutron logs for more information. [ 865.142995] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 865.142995] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 865.142995] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 865.142995] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 865.142995] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 865.142995] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 865.142995] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 865.142995] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 865.142995] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 865.142995] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 865.142995] env[63593]: ERROR nova.compute.manager raise self.value [ 865.142995] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 865.142995] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 865.142995] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 865.142995] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 865.143351] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 865.143351] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 865.143351] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5, please check neutron logs for more information. [ 865.143351] env[63593]: ERROR nova.compute.manager [ 865.143351] env[63593]: Traceback (most recent call last): [ 865.143351] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 865.143351] env[63593]: listener.cb(fileno) [ 865.143351] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 865.143351] env[63593]: result = function(*args, **kwargs) [ 865.143351] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 865.143351] env[63593]: return func(*args, **kwargs) [ 865.143351] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 865.143351] env[63593]: raise e [ 865.143351] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 865.143351] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 865.143351] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 865.143351] env[63593]: created_port_ids = self._update_ports_for_instance( [ 865.143351] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 865.143351] env[63593]: with excutils.save_and_reraise_exception(): [ 865.143351] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 865.143351] env[63593]: self.force_reraise() [ 865.143351] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 865.143351] env[63593]: raise self.value [ 865.143351] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 865.143351] env[63593]: updated_port = self._update_port( [ 865.143351] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 865.143351] env[63593]: _ensure_no_port_binding_failure(port) [ 865.143351] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 865.143351] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 865.143954] env[63593]: nova.exception.PortBindingFailed: Binding failed for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5, please check neutron logs for more information. [ 865.143954] env[63593]: Removing descriptor: 18 [ 865.262282] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 865.287774] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 865.288026] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 865.288185] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 865.288671] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 865.288671] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 865.288757] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 865.289643] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 865.289879] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 865.290079] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 865.290253] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 865.290430] env[63593]: DEBUG nova.virt.hardware [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 865.291601] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26db8eb2-f68b-4a3a-b022-ffc2f51bf870 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 865.301481] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84dbfb76-6ac2-4cff-ad32-380aa1782efc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 865.316722] env[63593]: ERROR nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5, please check neutron logs for more information. [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Traceback (most recent call last): [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] yield resources [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self.driver.spawn(context, instance, image_meta, [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] vm_ref = self.build_virtual_machine(instance, [ 865.316722] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] vif_infos = vmwarevif.get_vif_info(self._session, [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] for vif in network_info: [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] return self._sync_wrapper(fn, *args, **kwargs) [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self.wait() [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self[:] = self._gt.wait() [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] return self._exit_event.wait() [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 865.317060] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] current.throw(*self._exc) [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] result = function(*args, **kwargs) [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] return func(*args, **kwargs) [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] raise e [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] nwinfo = self.network_api.allocate_for_instance( [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] created_port_ids = self._update_ports_for_instance( [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] with excutils.save_and_reraise_exception(): [ 865.317383] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self.force_reraise() [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] raise self.value [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] updated_port = self._update_port( [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] _ensure_no_port_binding_failure(port) [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] raise exception.PortBindingFailed(port_id=port['id']) [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] nova.exception.PortBindingFailed: Binding failed for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5, please check neutron logs for more information. [ 865.317756] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] [ 865.317756] env[63593]: INFO nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Terminating instance [ 865.319095] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "refresh_cache-9385f942-728d-4041-bc52-77e3c9752b7f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 865.319250] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquired lock "refresh_cache-9385f942-728d-4041-bc52-77e3c9752b7f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 865.319414] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 865.351372] env[63593]: DEBUG nova.network.neutron [-] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 865.624712] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.386s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 865.625255] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 865.627774] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 28.633s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 865.836355] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 865.853641] env[63593]: INFO nova.compute.manager [-] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Took 1.02 seconds to deallocate network for instance. [ 865.855985] env[63593]: DEBUG nova.compute.claims [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 865.856216] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 865.919667] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 866.136142] env[63593]: DEBUG nova.compute.utils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 866.137705] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 866.137874] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 866.215423] env[63593]: DEBUG nova.policy [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6880a0273fa34ddb8d4f32069f8b0977', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'aa7ef13c10004d26a37ddb89cfebda6f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 866.423451] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Releasing lock "refresh_cache-9385f942-728d-4041-bc52-77e3c9752b7f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 866.423870] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 866.424070] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 866.425393] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-666d520f-964d-459a-bb75-616ec136daa9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.436657] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e869e4ff-b205-4302-8f47-10c08435da97 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.464711] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9385f942-728d-4041-bc52-77e3c9752b7f could not be found. [ 866.464979] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 866.465213] env[63593]: INFO nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 866.466247] env[63593]: DEBUG oslo.service.loopingcall [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 866.468252] env[63593]: DEBUG nova.compute.manager [-] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 866.468441] env[63593]: DEBUG nova.network.neutron [-] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 866.485967] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d109b3ea-af1c-414d-88cf-e8c84cc2348c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.493712] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3e10e94-1052-472d-af3e-9a707afc67d9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.497433] env[63593]: DEBUG nova.network.neutron [-] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 866.533356] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0bee393e-5f9b-436e-9a95-1515c3e10d09 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.537594] env[63593]: DEBUG nova.compute.manager [req-c88600e3-ae31-416f-95d2-af76422a4c05 req-b9466eaa-b27f-47e9-be1d-b4ff59b7ea3a service nova] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Received event network-changed-b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 866.537783] env[63593]: DEBUG nova.compute.manager [req-c88600e3-ae31-416f-95d2-af76422a4c05 req-b9466eaa-b27f-47e9-be1d-b4ff59b7ea3a service nova] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Refreshing instance network info cache due to event network-changed-b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 866.538057] env[63593]: DEBUG oslo_concurrency.lockutils [req-c88600e3-ae31-416f-95d2-af76422a4c05 req-b9466eaa-b27f-47e9-be1d-b4ff59b7ea3a service nova] Acquiring lock "refresh_cache-9385f942-728d-4041-bc52-77e3c9752b7f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 866.538140] env[63593]: DEBUG oslo_concurrency.lockutils [req-c88600e3-ae31-416f-95d2-af76422a4c05 req-b9466eaa-b27f-47e9-be1d-b4ff59b7ea3a service nova] Acquired lock "refresh_cache-9385f942-728d-4041-bc52-77e3c9752b7f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 866.538311] env[63593]: DEBUG nova.network.neutron [req-c88600e3-ae31-416f-95d2-af76422a4c05 req-b9466eaa-b27f-47e9-be1d-b4ff59b7ea3a service nova] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Refreshing network info cache for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 866.547469] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd4f4fd4-6776-4e9b-ab9b-4653a2d55bdc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.562147] env[63593]: DEBUG nova.compute.provider_tree [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 866.641238] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 866.915166] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Successfully created port: a5099b6d-a18d-4931-99fc-55803037d864 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 867.002284] env[63593]: DEBUG nova.network.neutron [-] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 867.060404] env[63593]: DEBUG nova.network.neutron [req-c88600e3-ae31-416f-95d2-af76422a4c05 req-b9466eaa-b27f-47e9-be1d-b4ff59b7ea3a service nova] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 867.066891] env[63593]: DEBUG nova.scheduler.client.report [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 867.173887] env[63593]: DEBUG nova.network.neutron [req-c88600e3-ae31-416f-95d2-af76422a4c05 req-b9466eaa-b27f-47e9-be1d-b4ff59b7ea3a service nova] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 867.505522] env[63593]: INFO nova.compute.manager [-] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Took 1.04 seconds to deallocate network for instance. [ 867.508081] env[63593]: DEBUG nova.compute.claims [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 867.508328] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 867.576762] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.949s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 867.577429] env[63593]: ERROR nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 075296e6-0c1e-4d55-a362-2fe91040eefb, please check neutron logs for more information. [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Traceback (most recent call last): [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self.driver.spawn(context, instance, image_meta, [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] vm_ref = self.build_virtual_machine(instance, [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] vif_infos = vmwarevif.get_vif_info(self._session, [ 867.577429] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] for vif in network_info: [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] return self._sync_wrapper(fn, *args, **kwargs) [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self.wait() [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self[:] = self._gt.wait() [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] return self._exit_event.wait() [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] result = hub.switch() [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 867.577741] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] return self.greenlet.switch() [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] result = function(*args, **kwargs) [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] return func(*args, **kwargs) [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] raise e [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] nwinfo = self.network_api.allocate_for_instance( [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] created_port_ids = self._update_ports_for_instance( [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] with excutils.save_and_reraise_exception(): [ 867.578083] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] self.force_reraise() [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] raise self.value [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] updated_port = self._update_port( [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] _ensure_no_port_binding_failure(port) [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] raise exception.PortBindingFailed(port_id=port['id']) [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] nova.exception.PortBindingFailed: Binding failed for port 075296e6-0c1e-4d55-a362-2fe91040eefb, please check neutron logs for more information. [ 867.578451] env[63593]: ERROR nova.compute.manager [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] [ 867.578731] env[63593]: DEBUG nova.compute.utils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Binding failed for port 075296e6-0c1e-4d55-a362-2fe91040eefb, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 867.582622] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 18.060s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 867.582796] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 867.582949] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 867.583250] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.158s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 867.586270] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Build of instance 94486acf-7f7b-4187-b9d1-332f70488bd5 was re-scheduled: Binding failed for port 075296e6-0c1e-4d55-a362-2fe91040eefb, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 867.586747] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 867.586983] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Acquiring lock "refresh_cache-94486acf-7f7b-4187-b9d1-332f70488bd5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 867.587142] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Acquired lock "refresh_cache-94486acf-7f7b-4187-b9d1-332f70488bd5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 867.587300] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 867.588756] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df7da431-8edd-4331-afb0-5d69994025a4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.599957] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-172727ca-a136-4504-abb0-96d397a83550 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.615849] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-948767f2-31e4-4841-9911-39fb75cc15b7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.622175] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e41e5d77-9499-4667-a384-6573ab53b554 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.655675] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 867.657716] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181433MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 867.657876] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 867.678624] env[63593]: DEBUG oslo_concurrency.lockutils [req-c88600e3-ae31-416f-95d2-af76422a4c05 req-b9466eaa-b27f-47e9-be1d-b4ff59b7ea3a service nova] Releasing lock "refresh_cache-9385f942-728d-4041-bc52-77e3c9752b7f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 867.678854] env[63593]: DEBUG nova.compute.manager [req-c88600e3-ae31-416f-95d2-af76422a4c05 req-b9466eaa-b27f-47e9-be1d-b4ff59b7ea3a service nova] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Received event network-vif-deleted-b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 867.685550] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 867.686091] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 867.686234] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 867.686414] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 867.686555] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 867.686697] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 867.686898] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 867.687065] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 867.687234] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 867.687394] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 867.687560] env[63593]: DEBUG nova.virt.hardware [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 867.688553] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-596283ec-d8fa-4f18-8225-651ca25e32ad {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.696189] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7cce9d93-4d61-4d06-bd49-72b4a4071a00 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.113574] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 868.226320] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 868.318520] env[63593]: ERROR nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a5099b6d-a18d-4931-99fc-55803037d864, please check neutron logs for more information. [ 868.318520] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 868.318520] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 868.318520] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 868.318520] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 868.318520] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 868.318520] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 868.318520] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 868.318520] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 868.318520] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 868.318520] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 868.318520] env[63593]: ERROR nova.compute.manager raise self.value [ 868.318520] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 868.318520] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 868.318520] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 868.318520] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 868.318901] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 868.318901] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 868.318901] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a5099b6d-a18d-4931-99fc-55803037d864, please check neutron logs for more information. [ 868.318901] env[63593]: ERROR nova.compute.manager [ 868.318901] env[63593]: Traceback (most recent call last): [ 868.318901] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 868.318901] env[63593]: listener.cb(fileno) [ 868.318901] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 868.318901] env[63593]: result = function(*args, **kwargs) [ 868.318901] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 868.318901] env[63593]: return func(*args, **kwargs) [ 868.318901] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 868.318901] env[63593]: raise e [ 868.318901] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 868.318901] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 868.318901] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 868.318901] env[63593]: created_port_ids = self._update_ports_for_instance( [ 868.318901] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 868.318901] env[63593]: with excutils.save_and_reraise_exception(): [ 868.318901] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 868.318901] env[63593]: self.force_reraise() [ 868.318901] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 868.318901] env[63593]: raise self.value [ 868.318901] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 868.318901] env[63593]: updated_port = self._update_port( [ 868.318901] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 868.318901] env[63593]: _ensure_no_port_binding_failure(port) [ 868.318901] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 868.318901] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 868.319573] env[63593]: nova.exception.PortBindingFailed: Binding failed for port a5099b6d-a18d-4931-99fc-55803037d864, please check neutron logs for more information. [ 868.319573] env[63593]: Removing descriptor: 18 [ 868.319573] env[63593]: ERROR nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a5099b6d-a18d-4931-99fc-55803037d864, please check neutron logs for more information. [ 868.319573] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Traceback (most recent call last): [ 868.319573] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 868.319573] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] yield resources [ 868.319573] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 868.319573] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self.driver.spawn(context, instance, image_meta, [ 868.319573] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 868.319573] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 868.319573] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 868.319573] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] vm_ref = self.build_virtual_machine(instance, [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] vif_infos = vmwarevif.get_vif_info(self._session, [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] for vif in network_info: [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] return self._sync_wrapper(fn, *args, **kwargs) [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self.wait() [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self[:] = self._gt.wait() [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] return self._exit_event.wait() [ 868.319844] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] result = hub.switch() [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] return self.greenlet.switch() [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] result = function(*args, **kwargs) [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] return func(*args, **kwargs) [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] raise e [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] nwinfo = self.network_api.allocate_for_instance( [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 868.320137] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] created_port_ids = self._update_ports_for_instance( [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] with excutils.save_and_reraise_exception(): [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self.force_reraise() [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] raise self.value [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] updated_port = self._update_port( [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] _ensure_no_port_binding_failure(port) [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 868.320419] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] raise exception.PortBindingFailed(port_id=port['id']) [ 868.320677] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] nova.exception.PortBindingFailed: Binding failed for port a5099b6d-a18d-4931-99fc-55803037d864, please check neutron logs for more information. [ 868.320677] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] [ 868.320677] env[63593]: INFO nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Terminating instance [ 868.320677] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "refresh_cache-2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 868.320677] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquired lock "refresh_cache-2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 868.320677] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 868.545412] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8236e209-4840-4ca2-b30d-c7e753cebbd1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.553433] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a982d3bc-c2d4-495c-9a28-4eff8740f23a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.558424] env[63593]: DEBUG nova.compute.manager [req-0c7621f8-6e42-4934-8999-8b43b8a21bdd req-7d1de82c-dc10-4401-8705-6f87d5445c6a service nova] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Received event network-changed-a5099b6d-a18d-4931-99fc-55803037d864 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 868.558614] env[63593]: DEBUG nova.compute.manager [req-0c7621f8-6e42-4934-8999-8b43b8a21bdd req-7d1de82c-dc10-4401-8705-6f87d5445c6a service nova] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Refreshing instance network info cache due to event network-changed-a5099b6d-a18d-4931-99fc-55803037d864. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 868.558800] env[63593]: DEBUG oslo_concurrency.lockutils [req-0c7621f8-6e42-4934-8999-8b43b8a21bdd req-7d1de82c-dc10-4401-8705-6f87d5445c6a service nova] Acquiring lock "refresh_cache-2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 868.588147] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9fa70e75-abae-4097-b047-751289ee0338 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.595718] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c525f325-0aab-43c0-ac3b-f6b95f7bd993 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.609705] env[63593]: DEBUG nova.compute.provider_tree [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 868.695815] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 868.696047] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 868.730194] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Releasing lock "refresh_cache-94486acf-7f7b-4187-b9d1-332f70488bd5" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 868.730520] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 868.730721] env[63593]: DEBUG nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 868.730886] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 868.744658] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 868.841969] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 868.910888] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 869.112488] env[63593]: DEBUG nova.scheduler.client.report [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 869.247275] env[63593]: DEBUG nova.network.neutron [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 869.413774] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Releasing lock "refresh_cache-2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 869.414280] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 869.414479] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 869.414808] env[63593]: DEBUG oslo_concurrency.lockutils [req-0c7621f8-6e42-4934-8999-8b43b8a21bdd req-7d1de82c-dc10-4401-8705-6f87d5445c6a service nova] Acquired lock "refresh_cache-2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 869.414978] env[63593]: DEBUG nova.network.neutron [req-0c7621f8-6e42-4934-8999-8b43b8a21bdd req-7d1de82c-dc10-4401-8705-6f87d5445c6a service nova] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Refreshing network info cache for port a5099b6d-a18d-4931-99fc-55803037d864 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 869.416075] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c9ea8465-e373-41b5-aa97-b9d9b3a4c405 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.426056] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-165ca7c6-d8d2-4285-880d-a5f07c70f1c0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.449294] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8 could not be found. [ 869.449554] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 869.449733] env[63593]: INFO nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Took 0.04 seconds to destroy the instance on the hypervisor. [ 869.449974] env[63593]: DEBUG oslo.service.loopingcall [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 869.450214] env[63593]: DEBUG nova.compute.manager [-] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 869.450310] env[63593]: DEBUG nova.network.neutron [-] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 869.467545] env[63593]: DEBUG nova.network.neutron [-] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 869.617255] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.034s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 869.617876] env[63593]: ERROR nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3b25f8af-5d71-4ca2-80fe-c544d5569384, please check neutron logs for more information. [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Traceback (most recent call last): [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self.driver.spawn(context, instance, image_meta, [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self._vmops.spawn(context, instance, image_meta, injected_files, [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] vm_ref = self.build_virtual_machine(instance, [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] vif_infos = vmwarevif.get_vif_info(self._session, [ 869.617876] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] for vif in network_info: [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] return self._sync_wrapper(fn, *args, **kwargs) [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self.wait() [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self[:] = self._gt.wait() [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] return self._exit_event.wait() [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] current.throw(*self._exc) [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 869.618189] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] result = function(*args, **kwargs) [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] return func(*args, **kwargs) [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] raise e [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] nwinfo = self.network_api.allocate_for_instance( [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] created_port_ids = self._update_ports_for_instance( [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] with excutils.save_and_reraise_exception(): [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] self.force_reraise() [ 869.618594] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 869.618920] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] raise self.value [ 869.618920] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 869.618920] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] updated_port = self._update_port( [ 869.618920] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 869.618920] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] _ensure_no_port_binding_failure(port) [ 869.618920] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 869.618920] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] raise exception.PortBindingFailed(port_id=port['id']) [ 869.618920] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] nova.exception.PortBindingFailed: Binding failed for port 3b25f8af-5d71-4ca2-80fe-c544d5569384, please check neutron logs for more information. [ 869.618920] env[63593]: ERROR nova.compute.manager [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] [ 869.618920] env[63593]: DEBUG nova.compute.utils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Binding failed for port 3b25f8af-5d71-4ca2-80fe-c544d5569384, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 869.620764] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 10.309s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 869.622689] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Build of instance 43b2aa92-913e-4bae-9e25-82cf423d4148 was re-scheduled: Binding failed for port 3b25f8af-5d71-4ca2-80fe-c544d5569384, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 869.623102] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 869.623327] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Acquiring lock "refresh_cache-43b2aa92-913e-4bae-9e25-82cf423d4148" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 869.623472] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Acquired lock "refresh_cache-43b2aa92-913e-4bae-9e25-82cf423d4148" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 869.623629] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 869.750426] env[63593]: INFO nova.compute.manager [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] [instance: 94486acf-7f7b-4187-b9d1-332f70488bd5] Took 1.02 seconds to deallocate network for instance. [ 869.933698] env[63593]: DEBUG nova.network.neutron [req-0c7621f8-6e42-4934-8999-8b43b8a21bdd req-7d1de82c-dc10-4401-8705-6f87d5445c6a service nova] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 869.969849] env[63593]: DEBUG nova.network.neutron [-] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 870.018370] env[63593]: DEBUG nova.network.neutron [req-0c7621f8-6e42-4934-8999-8b43b8a21bdd req-7d1de82c-dc10-4401-8705-6f87d5445c6a service nova] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 870.141885] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 870.224956] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 870.415057] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86a99cb6-8edf-425a-b6b8-8da2e48adeb3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.422980] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff1ed74a-6d6b-4dc9-93ab-f70f38b0e302 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.452303] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3596b291-58be-4807-8d5d-41227a3e8aee {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.459313] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9701b79-21bd-4c6d-bbb3-d471b407ed0c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.472059] env[63593]: INFO nova.compute.manager [-] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Took 1.02 seconds to deallocate network for instance. [ 870.472537] env[63593]: DEBUG nova.compute.provider_tree [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 870.475209] env[63593]: DEBUG nova.compute.claims [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 870.475424] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 870.520968] env[63593]: DEBUG oslo_concurrency.lockutils [req-0c7621f8-6e42-4934-8999-8b43b8a21bdd req-7d1de82c-dc10-4401-8705-6f87d5445c6a service nova] Releasing lock "refresh_cache-2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 870.521292] env[63593]: DEBUG nova.compute.manager [req-0c7621f8-6e42-4934-8999-8b43b8a21bdd req-7d1de82c-dc10-4401-8705-6f87d5445c6a service nova] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Received event network-vif-deleted-a5099b6d-a18d-4931-99fc-55803037d864 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 870.728682] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Releasing lock "refresh_cache-43b2aa92-913e-4bae-9e25-82cf423d4148" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 870.728891] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 870.729086] env[63593]: DEBUG nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 870.729285] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 870.742765] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 870.784219] env[63593]: INFO nova.scheduler.client.report [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Deleted allocations for instance 94486acf-7f7b-4187-b9d1-332f70488bd5 [ 870.976228] env[63593]: DEBUG nova.scheduler.client.report [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 871.245228] env[63593]: DEBUG nova.network.neutron [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 871.291332] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2d7e5b67-5158-4038-8ec3-450b7936fca1 tempest-InstanceActionsTestJSON-1588012900 tempest-InstanceActionsTestJSON-1588012900-project-member] Lock "94486acf-7f7b-4187-b9d1-332f70488bd5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 151.973s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.481559] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.862s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.482165] env[63593]: ERROR nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989, please check neutron logs for more information. [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Traceback (most recent call last): [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self.driver.spawn(context, instance, image_meta, [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self._vmops.spawn(context, instance, image_meta, injected_files, [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] vm_ref = self.build_virtual_machine(instance, [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] vif_infos = vmwarevif.get_vif_info(self._session, [ 871.482165] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] for vif in network_info: [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] return self._sync_wrapper(fn, *args, **kwargs) [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self.wait() [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self[:] = self._gt.wait() [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] return self._exit_event.wait() [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] current.throw(*self._exc) [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 871.482430] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] result = function(*args, **kwargs) [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] return func(*args, **kwargs) [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] raise e [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] nwinfo = self.network_api.allocate_for_instance( [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] created_port_ids = self._update_ports_for_instance( [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] with excutils.save_and_reraise_exception(): [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] self.force_reraise() [ 871.482689] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 871.482958] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] raise self.value [ 871.482958] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 871.482958] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] updated_port = self._update_port( [ 871.482958] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 871.482958] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] _ensure_no_port_binding_failure(port) [ 871.482958] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 871.482958] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] raise exception.PortBindingFailed(port_id=port['id']) [ 871.482958] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] nova.exception.PortBindingFailed: Binding failed for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989, please check neutron logs for more information. [ 871.482958] env[63593]: ERROR nova.compute.manager [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] [ 871.482958] env[63593]: DEBUG nova.compute.utils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Binding failed for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 871.484605] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Build of instance be01dd8d-53fb-4fcc-9bf8-41934b4d0263 was re-scheduled: Binding failed for port 24aa41a9-9a5b-48d3-80b9-2b5cf03df989, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 871.484989] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 871.485226] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Acquiring lock "refresh_cache-be01dd8d-53fb-4fcc-9bf8-41934b4d0263" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 871.485370] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Acquired lock "refresh_cache-be01dd8d-53fb-4fcc-9bf8-41934b4d0263" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 871.485525] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 871.486488] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 10.883s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 871.747656] env[63593]: INFO nova.compute.manager [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] [instance: 43b2aa92-913e-4bae-9e25-82cf423d4148] Took 1.02 seconds to deallocate network for instance. [ 871.794173] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 872.012665] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 872.121800] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 872.255184] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2deb041c-e5cb-4500-a233-d86ae9462b22 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.263238] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5acb3dc-272a-4ca7-9201-d4278dbfc3c8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.295899] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0b1ad43-7317-4193-8767-4469ed328dc5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.309018] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16834dd8-ab77-4ffc-b453-05a4d1891e86 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.314363] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 872.322806] env[63593]: DEBUG nova.compute.provider_tree [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 872.624813] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Releasing lock "refresh_cache-be01dd8d-53fb-4fcc-9bf8-41934b4d0263" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 872.625073] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 872.625261] env[63593]: DEBUG nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 872.625427] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 872.639236] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 872.783976] env[63593]: INFO nova.scheduler.client.report [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Deleted allocations for instance 43b2aa92-913e-4bae-9e25-82cf423d4148 [ 872.827019] env[63593]: DEBUG nova.scheduler.client.report [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 873.141853] env[63593]: DEBUG nova.network.neutron [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 873.296328] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f08e1719-9ed9-462f-ab96-40763068c22c tempest-ImagesOneServerTestJSON-1877294267 tempest-ImagesOneServerTestJSON-1877294267-project-member] Lock "43b2aa92-913e-4bae-9e25-82cf423d4148" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 147.913s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 873.332196] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.845s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 873.332531] env[63593]: ERROR nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a573cab4-883b-4744-b3c9-c6590951a92c, please check neutron logs for more information. [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Traceback (most recent call last): [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self.driver.spawn(context, instance, image_meta, [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self._vmops.spawn(context, instance, image_meta, injected_files, [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] vm_ref = self.build_virtual_machine(instance, [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] vif_infos = vmwarevif.get_vif_info(self._session, [ 873.332531] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] for vif in network_info: [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] return self._sync_wrapper(fn, *args, **kwargs) [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self.wait() [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self[:] = self._gt.wait() [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] return self._exit_event.wait() [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] current.throw(*self._exc) [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 873.332819] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] result = function(*args, **kwargs) [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] return func(*args, **kwargs) [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] raise e [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] nwinfo = self.network_api.allocate_for_instance( [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] created_port_ids = self._update_ports_for_instance( [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] with excutils.save_and_reraise_exception(): [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] self.force_reraise() [ 873.333316] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.333639] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] raise self.value [ 873.333639] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 873.333639] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] updated_port = self._update_port( [ 873.333639] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.333639] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] _ensure_no_port_binding_failure(port) [ 873.333639] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.333639] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] raise exception.PortBindingFailed(port_id=port['id']) [ 873.333639] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] nova.exception.PortBindingFailed: Binding failed for port a573cab4-883b-4744-b3c9-c6590951a92c, please check neutron logs for more information. [ 873.333639] env[63593]: ERROR nova.compute.manager [instance: 2b478998-632e-4286-8d97-eab4f025bdda] [ 873.333639] env[63593]: DEBUG nova.compute.utils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Binding failed for port a573cab4-883b-4744-b3c9-c6590951a92c, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 873.335285] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.163s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 873.336785] env[63593]: INFO nova.compute.claims [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 873.339468] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Build of instance 2b478998-632e-4286-8d97-eab4f025bdda was re-scheduled: Binding failed for port a573cab4-883b-4744-b3c9-c6590951a92c, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 873.339912] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 873.340185] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "refresh_cache-2b478998-632e-4286-8d97-eab4f025bdda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 873.340334] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "refresh_cache-2b478998-632e-4286-8d97-eab4f025bdda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 873.340492] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 873.644648] env[63593]: INFO nova.compute.manager [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] [instance: be01dd8d-53fb-4fcc-9bf8-41934b4d0263] Took 1.02 seconds to deallocate network for instance. [ 873.802551] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 873.860381] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 873.943721] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 874.323150] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 874.447150] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "refresh_cache-2b478998-632e-4286-8d97-eab4f025bdda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 874.447597] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 874.447736] env[63593]: DEBUG nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 874.447938] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 874.469209] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 875.260398] env[63593]: DEBUG nova.network.neutron [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 875.296052] env[63593]: INFO nova.scheduler.client.report [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Deleted allocations for instance be01dd8d-53fb-4fcc-9bf8-41934b4d0263 [ 875.518276] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61a53655-313d-42a8-a68c-00ce78a468e7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 875.526053] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28a95937-df1d-4dc2-804a-6dbd89bdef21 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 875.559878] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d45b6992-5439-4fb0-8209-275fa65e3c70 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 875.568025] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c02e68dc-b3e3-482b-9cc4-78d5dc453575 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 875.578902] env[63593]: DEBUG nova.compute.provider_tree [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 875.766021] env[63593]: INFO nova.compute.manager [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 2b478998-632e-4286-8d97-eab4f025bdda] Took 1.32 seconds to deallocate network for instance. [ 875.809261] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cedc88f6-6b7e-45e6-a4f5-d0204e8e097a tempest-ServerGroupTestJSON-1213431026 tempest-ServerGroupTestJSON-1213431026-project-member] Lock "be01dd8d-53fb-4fcc-9bf8-41934b4d0263" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 147.099s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 876.081820] env[63593]: DEBUG nova.scheduler.client.report [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 876.312687] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 876.588986] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.254s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 876.589596] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 876.596044] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 10.739s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 876.822745] env[63593]: INFO nova.scheduler.client.report [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleted allocations for instance 2b478998-632e-4286-8d97-eab4f025bdda [ 876.852258] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 877.105555] env[63593]: DEBUG nova.compute.utils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 877.108779] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Not allocating networking since 'none' was specified. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 877.336500] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0add7608-94e6-41d3-9312-750f8bc38808 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "2b478998-632e-4286-8d97-eab4f025bdda" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 146.483s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 877.449054] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79463ca4-026f-4c1d-bdee-2a3dc1ebece5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 877.460198] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-340fa17c-7cce-4451-b1e6-20310c7a16e9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 877.499037] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42bba3c7-4a4e-4312-b994-095cb0ab278d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 877.507298] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1060c115-8578-422a-be97-7dbec15fb18f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 877.521710] env[63593]: DEBUG nova.compute.provider_tree [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 877.610364] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 877.839192] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 878.026449] env[63593]: DEBUG nova.scheduler.client.report [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 878.360280] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 878.479725] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d05df664-cf9b-4dd9-90de-1fa791a5346c tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquiring lock "0c9d6b46-e1f1-4dcc-b5bf-c6ca520d4e0f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 878.479973] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d05df664-cf9b-4dd9-90de-1fa791a5346c tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "0c9d6b46-e1f1-4dcc-b5bf-c6ca520d4e0f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 878.534554] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.939s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 878.535196] env[63593]: ERROR nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2c459dc7-c902-48f5-a548-588973a6e93b, please check neutron logs for more information. [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Traceback (most recent call last): [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self.driver.spawn(context, instance, image_meta, [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self._vmops.spawn(context, instance, image_meta, injected_files, [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] vm_ref = self.build_virtual_machine(instance, [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] vif_infos = vmwarevif.get_vif_info(self._session, [ 878.535196] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] for vif in network_info: [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] return self._sync_wrapper(fn, *args, **kwargs) [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self.wait() [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self[:] = self._gt.wait() [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] return self._exit_event.wait() [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] current.throw(*self._exc) [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 878.535449] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] result = function(*args, **kwargs) [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] return func(*args, **kwargs) [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] raise e [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] nwinfo = self.network_api.allocate_for_instance( [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] created_port_ids = self._update_ports_for_instance( [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] with excutils.save_and_reraise_exception(): [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] self.force_reraise() [ 878.535739] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 878.536061] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] raise self.value [ 878.536061] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 878.536061] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] updated_port = self._update_port( [ 878.536061] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 878.536061] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] _ensure_no_port_binding_failure(port) [ 878.536061] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 878.536061] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] raise exception.PortBindingFailed(port_id=port['id']) [ 878.536061] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] nova.exception.PortBindingFailed: Binding failed for port 2c459dc7-c902-48f5-a548-588973a6e93b, please check neutron logs for more information. [ 878.536061] env[63593]: ERROR nova.compute.manager [instance: 464b651f-3f3f-425d-9da6-546ea9610515] [ 878.536061] env[63593]: DEBUG nova.compute.utils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Binding failed for port 2c459dc7-c902-48f5-a548-588973a6e93b, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 878.537159] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.029s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 878.543037] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Build of instance 464b651f-3f3f-425d-9da6-546ea9610515 was re-scheduled: Binding failed for port 2c459dc7-c902-48f5-a548-588973a6e93b, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 878.543037] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 878.543037] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Acquiring lock "refresh_cache-464b651f-3f3f-425d-9da6-546ea9610515" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 878.543037] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Acquired lock "refresh_cache-464b651f-3f3f-425d-9da6-546ea9610515" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 878.543250] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 878.622183] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 878.658312] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 878.658676] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 878.658962] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 878.659242] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 878.659479] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 878.659788] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 878.660039] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 878.660204] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 878.660370] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 878.660530] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 878.660697] env[63593]: DEBUG nova.virt.hardware [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 878.664024] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7694fde8-ad90-4410-b3ca-0dd76345b738 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.671009] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-151646a9-dbe9-41ad-aab5-926314cbb40e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.686143] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Instance VIF info [] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 878.693425] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Creating folder: Project (82d41e9d7d28498ebc7691d3a2b6012f). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 878.693425] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-defa58e8-2707-4938-89b6-bc25c3dc288a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.704091] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Created folder: Project (82d41e9d7d28498ebc7691d3a2b6012f) in parent group-v291016. [ 878.704159] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Creating folder: Instances. Parent ref: group-v291035. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 878.704389] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d0e7b2bb-94d2-4c8b-8813-f39cde0a2f22 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.712994] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Created folder: Instances in parent group-v291035. [ 878.713252] env[63593]: DEBUG oslo.service.loopingcall [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 878.713437] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 878.713635] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-69fdc262-b26c-4cf6-b4ca-a3b84f175de9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.731675] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 878.731675] env[63593]: value = "task-1367967" [ 878.731675] env[63593]: _type = "Task" [ 878.731675] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 878.741718] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367967, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 879.076575] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 879.192358] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 879.242167] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367967, 'name': CreateVM_Task, 'duration_secs': 0.297061} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 879.242368] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 879.242805] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 879.242950] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 879.243324] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 879.243580] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b65d43dc-7530-4c6b-9fc1-470be377194d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.248314] env[63593]: DEBUG oslo_vmware.api [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Waiting for the task: (returnval){ [ 879.248314] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52266a91-d5f6-4812-27ae-746b76dad3ff" [ 879.248314] env[63593]: _type = "Task" [ 879.248314] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 879.259378] env[63593]: DEBUG oslo_vmware.api [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52266a91-d5f6-4812-27ae-746b76dad3ff, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 879.386620] env[63593]: DEBUG oslo_concurrency.lockutils [None req-48c5bd4b-ae92-4455-ad4e-0909267f969d tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "4dd65fb0-e079-4520-8809-dfd275d876f7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 879.387505] env[63593]: DEBUG oslo_concurrency.lockutils [None req-48c5bd4b-ae92-4455-ad4e-0909267f969d tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "4dd65fb0-e079-4520-8809-dfd275d876f7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 879.484401] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83f9e121-bfa9-4ed3-a3ac-bf392c7e5fbb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.489397] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a295e68-4bdf-41ce-b15e-6bf84e48cbcc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.518177] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11338a34-b10b-4ace-b5dc-028ec413c3a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.525557] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16b547e3-3488-4517-a692-8024fd6ba100 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.537820] env[63593]: DEBUG nova.compute.provider_tree [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 879.695831] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Releasing lock "refresh_cache-464b651f-3f3f-425d-9da6-546ea9610515" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 879.695831] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 879.696020] env[63593]: DEBUG nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 879.696208] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 879.714091] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 879.762569] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 879.762986] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 879.763351] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 880.041071] env[63593]: DEBUG nova.scheduler.client.report [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 880.216916] env[63593]: DEBUG nova.network.neutron [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 880.547138] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.010s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.548155] env[63593]: ERROR nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5, please check neutron logs for more information. [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Traceback (most recent call last): [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self.driver.spawn(context, instance, image_meta, [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] vm_ref = self.build_virtual_machine(instance, [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] vif_infos = vmwarevif.get_vif_info(self._session, [ 880.548155] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] for vif in network_info: [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] return self._sync_wrapper(fn, *args, **kwargs) [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self.wait() [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self[:] = self._gt.wait() [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] return self._exit_event.wait() [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] current.throw(*self._exc) [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 880.548510] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] result = function(*args, **kwargs) [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] return func(*args, **kwargs) [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] raise e [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] nwinfo = self.network_api.allocate_for_instance( [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] created_port_ids = self._update_ports_for_instance( [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] with excutils.save_and_reraise_exception(): [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] self.force_reraise() [ 880.548838] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 880.549175] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] raise self.value [ 880.549175] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 880.549175] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] updated_port = self._update_port( [ 880.549175] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 880.549175] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] _ensure_no_port_binding_failure(port) [ 880.549175] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 880.549175] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] raise exception.PortBindingFailed(port_id=port['id']) [ 880.549175] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] nova.exception.PortBindingFailed: Binding failed for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5, please check neutron logs for more information. [ 880.549175] env[63593]: ERROR nova.compute.manager [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] [ 880.549401] env[63593]: DEBUG nova.compute.utils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Binding failed for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 880.550962] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 12.893s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.553479] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Build of instance 9385f942-728d-4041-bc52-77e3c9752b7f was re-scheduled: Binding failed for port b25f57f8-8f75-43dd-8253-fc0b1cb3a1d5, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 880.554147] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 880.554504] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "refresh_cache-9385f942-728d-4041-bc52-77e3c9752b7f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 880.554753] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquired lock "refresh_cache-9385f942-728d-4041-bc52-77e3c9752b7f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 880.555033] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 880.719373] env[63593]: INFO nova.compute.manager [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 464b651f-3f3f-425d-9da6-546ea9610515] Took 1.02 seconds to deallocate network for instance. [ 881.079118] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 881.200486] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 881.579058] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 94ff0182-d1ce-405f-bdb5-a2399bd0f4de actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 881.579310] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e312fd5-01bf-4bd4-8a61-26624ae05036 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 881.650254] env[63593]: DEBUG oslo_concurrency.lockutils [None req-96b8f73d-4a6f-4d88-8937-917863b694fb tempest-ServerMetadataNegativeTestJSON-2066118354 tempest-ServerMetadataNegativeTestJSON-2066118354-project-member] Acquiring lock "a4c51198-1e5d-4a59-8a9b-a997872e372b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 881.650254] env[63593]: DEBUG oslo_concurrency.lockutils [None req-96b8f73d-4a6f-4d88-8937-917863b694fb tempest-ServerMetadataNegativeTestJSON-2066118354 tempest-ServerMetadataNegativeTestJSON-2066118354-project-member] Lock "a4c51198-1e5d-4a59-8a9b-a997872e372b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 881.705605] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Releasing lock "refresh_cache-9385f942-728d-4041-bc52-77e3c9752b7f" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 881.705902] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 881.706141] env[63593]: DEBUG nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 881.706321] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 881.721952] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 881.749317] env[63593]: INFO nova.scheduler.client.report [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Deleted allocations for instance 464b651f-3f3f-425d-9da6-546ea9610515 [ 882.086023] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 464b651f-3f3f-425d-9da6-546ea9610515 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 882.224708] env[63593]: DEBUG nova.network.neutron [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 882.256744] env[63593]: DEBUG oslo_concurrency.lockutils [None req-08904c2f-f98a-4611-824e-8fd27a393dd6 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "464b651f-3f3f-425d-9da6-546ea9610515" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 150.511s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.589874] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 9385f942-728d-4041-bc52-77e3c9752b7f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 882.589874] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 882.589874] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 991481e2-557e-4222-adfc-d298fa9314a0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 882.730123] env[63593]: INFO nova.compute.manager [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 9385f942-728d-4041-bc52-77e3c9752b7f] Took 1.02 seconds to deallocate network for instance. [ 882.759737] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 883.095138] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance ef8646c6-7ead-4c9f-8e8f-85c758a35006 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 883.287781] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 883.599999] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a223dd01-d07d-4ac5-acfc-2e03b4c7dff4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 883.767127] env[63593]: INFO nova.scheduler.client.report [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Deleted allocations for instance 9385f942-728d-4041-bc52-77e3c9752b7f [ 884.103369] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d3a4902f-1829-427c-9f37-6e83ae7265e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 884.276325] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d48eb9b9-6c79-46cc-a3c4-1531ebb016eb tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "9385f942-728d-4041-bc52-77e3c9752b7f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 137.531s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 884.538604] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a96bc521-cec1-413b-84df-d6323f356c82 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Acquiring lock "0cdcc9d3-9963-49b5-8744-241b865335e2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 884.538831] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a96bc521-cec1-413b-84df-d6323f356c82 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "0cdcc9d3-9963-49b5-8744-241b865335e2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 884.607178] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 884.779609] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 885.110859] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fbd620a6-3743-4737-98b7-b13928b6587d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 885.300924] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 885.613551] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 885.826353] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3bd3c13f-ce31-468b-b893-2e083f8084ba tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Acquiring lock "38e38231-7b05-43c7-8de2-993a03ed4f57" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 885.826689] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3bd3c13f-ce31-468b-b893-2e083f8084ba tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "38e38231-7b05-43c7-8de2-993a03ed4f57" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 886.117693] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 328b91c6-4ae4-442f-ac87-519658a812ff has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 886.621447] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 887.123889] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 887.628038] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 888.133541] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 888.636328] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d760b552-e2ff-46db-88eb-f5f58fc88239 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 889.139735] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 257c613c-8613-42e5-b400-b9c352a151f0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 889.643056] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 890.147903] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0c9d6b46-e1f1-4dcc-b5bf-c6ca520d4e0f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 890.651516] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4dd65fb0-e079-4520-8809-dfd275d876f7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 891.153906] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a4c51198-1e5d-4a59-8a9b-a997872e372b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 891.154195] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 4 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 891.154341] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1280MB phys_disk=200GB used_disk=4GB total_vcpus=48 used_vcpus=4 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 891.406610] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-315f872f-7e8b-4ec3-bf8c-03aac8991ab9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.414051] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a16a463-c8df-4c5c-876a-97f720a4c7e4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.442399] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a0b9765-8e8b-4839-8f23-cba871b81774 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.448974] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c916d29-32a2-41fd-b0b2-ae4007aa730d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.461716] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 891.964287] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 892.472100] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 892.472383] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 11.921s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 892.472684] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 21.997s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 893.224970] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a86e6f4f-981a-4009-8d50-b417acc00644 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 893.232495] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e84e95d7-c651-48d0-9d41-f61b6e5bcb24 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 893.262629] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77ccceea-9736-4c83-8050-242884abc81d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 893.269909] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33ad89af-ba43-4bd3-aee4-7d29aeb2b03b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 893.283523] env[63593]: DEBUG nova.compute.provider_tree [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 893.786305] env[63593]: DEBUG nova.scheduler.client.report [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 894.290822] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.818s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 894.291546] env[63593]: ERROR nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a5099b6d-a18d-4931-99fc-55803037d864, please check neutron logs for more information. [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Traceback (most recent call last): [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self.driver.spawn(context, instance, image_meta, [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] vm_ref = self.build_virtual_machine(instance, [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] vif_infos = vmwarevif.get_vif_info(self._session, [ 894.291546] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] for vif in network_info: [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] return self._sync_wrapper(fn, *args, **kwargs) [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self.wait() [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self[:] = self._gt.wait() [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] return self._exit_event.wait() [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] result = hub.switch() [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 894.291871] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] return self.greenlet.switch() [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] result = function(*args, **kwargs) [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] return func(*args, **kwargs) [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] raise e [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] nwinfo = self.network_api.allocate_for_instance( [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] created_port_ids = self._update_ports_for_instance( [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] with excutils.save_and_reraise_exception(): [ 894.292221] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] self.force_reraise() [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] raise self.value [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] updated_port = self._update_port( [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] _ensure_no_port_binding_failure(port) [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] raise exception.PortBindingFailed(port_id=port['id']) [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] nova.exception.PortBindingFailed: Binding failed for port a5099b6d-a18d-4931-99fc-55803037d864, please check neutron logs for more information. [ 894.292551] env[63593]: ERROR nova.compute.manager [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] [ 894.292837] env[63593]: DEBUG nova.compute.utils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Binding failed for port a5099b6d-a18d-4931-99fc-55803037d864, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 894.293504] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.979s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 894.294928] env[63593]: INFO nova.compute.claims [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 894.297788] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Build of instance 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8 was re-scheduled: Binding failed for port a5099b6d-a18d-4931-99fc-55803037d864, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 894.298208] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 894.298431] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquiring lock "refresh_cache-2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 894.298610] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Acquired lock "refresh_cache-2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 894.298780] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 894.836694] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 894.924522] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 895.427266] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Releasing lock "refresh_cache-2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 895.427530] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 895.427666] env[63593]: DEBUG nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 895.427834] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 895.446455] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 895.544307] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba985999-440b-4139-940b-b44db9029c4d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 895.551606] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1547c4ce-02e4-44aa-9600-742ae3ff136c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 895.579983] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f610daf-8f88-4f80-aef5-f574dfcf1b29 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 895.586510] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8fbb3fbe-18cf-4fbb-9928-20a544621eeb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 895.599158] env[63593]: DEBUG nova.compute.provider_tree [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 895.949402] env[63593]: DEBUG nova.network.neutron [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 896.102211] env[63593]: DEBUG nova.scheduler.client.report [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 896.452836] env[63593]: INFO nova.compute.manager [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] [instance: 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8] Took 1.02 seconds to deallocate network for instance. [ 896.607383] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.314s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 896.607935] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 896.610500] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 22.288s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 896.611882] env[63593]: INFO nova.compute.claims [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 897.116444] env[63593]: DEBUG nova.compute.utils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 897.124668] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 897.124844] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 897.198573] env[63593]: DEBUG nova.policy [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ca3adaa6d5744600902f345db2737797', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd92d70a20b9348cdaacadc6f210c1963', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 897.476316] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Successfully created port: fdb0f1c9-485b-4b15-94d3-e7d274b52f11 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 897.501028] env[63593]: INFO nova.scheduler.client.report [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Deleted allocations for instance 2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8 [ 897.629597] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 897.906112] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4d277f1-52dd-4e45-9860-207a47245f7f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.914396] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b999a219-1c18-4905-81e0-c3cdcad89ea7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.955603] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-324cb7bc-35ed-41c9-8de6-2f1cd2d7f9c0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.963154] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33c40d20-289c-4171-89f4-fe8a322aa76b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 897.978626] env[63593]: DEBUG nova.compute.provider_tree [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 898.016821] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2492ee7c-cf69-4ed1-add2-1db80dd8d766 tempest-AttachVolumeTestJSON-869063485 tempest-AttachVolumeTestJSON-869063485-project-member] Lock "2f7b5f49-ca72-456d-9dd0-a495fb7cdcb8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 133.160s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.279524] env[63593]: DEBUG nova.compute.manager [req-50763e42-3f24-416d-a8ac-d577f3155138 req-7e04bf95-4156-46dc-964a-04266cae6c88 service nova] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Received event network-changed-fdb0f1c9-485b-4b15-94d3-e7d274b52f11 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 898.279524] env[63593]: DEBUG nova.compute.manager [req-50763e42-3f24-416d-a8ac-d577f3155138 req-7e04bf95-4156-46dc-964a-04266cae6c88 service nova] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Refreshing instance network info cache due to event network-changed-fdb0f1c9-485b-4b15-94d3-e7d274b52f11. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 898.279705] env[63593]: DEBUG oslo_concurrency.lockutils [req-50763e42-3f24-416d-a8ac-d577f3155138 req-7e04bf95-4156-46dc-964a-04266cae6c88 service nova] Acquiring lock "refresh_cache-ef8646c6-7ead-4c9f-8e8f-85c758a35006" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 898.279820] env[63593]: DEBUG oslo_concurrency.lockutils [req-50763e42-3f24-416d-a8ac-d577f3155138 req-7e04bf95-4156-46dc-964a-04266cae6c88 service nova] Acquired lock "refresh_cache-ef8646c6-7ead-4c9f-8e8f-85c758a35006" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 898.279983] env[63593]: DEBUG nova.network.neutron [req-50763e42-3f24-416d-a8ac-d577f3155138 req-7e04bf95-4156-46dc-964a-04266cae6c88 service nova] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Refreshing network info cache for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 898.479590] env[63593]: DEBUG nova.scheduler.client.report [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 898.488469] env[63593]: ERROR nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11, please check neutron logs for more information. [ 898.488469] env[63593]: ERROR nova.compute.manager Traceback (most recent call last): [ 898.488469] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 898.488469] env[63593]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 898.488469] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 898.488469] env[63593]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 898.488469] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 898.488469] env[63593]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 898.488469] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 898.488469] env[63593]: ERROR nova.compute.manager self.force_reraise() [ 898.488469] env[63593]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 898.488469] env[63593]: ERROR nova.compute.manager raise self.value [ 898.488469] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 898.488469] env[63593]: ERROR nova.compute.manager updated_port = self._update_port( [ 898.488469] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 898.488469] env[63593]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 898.489024] env[63593]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 898.489024] env[63593]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 898.489024] env[63593]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11, please check neutron logs for more information. [ 898.489024] env[63593]: ERROR nova.compute.manager [ 898.489024] env[63593]: Traceback (most recent call last): [ 898.489024] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 898.489024] env[63593]: listener.cb(fileno) [ 898.489024] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 898.489024] env[63593]: result = function(*args, **kwargs) [ 898.489024] env[63593]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 898.489024] env[63593]: return func(*args, **kwargs) [ 898.489024] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 898.489024] env[63593]: raise e [ 898.489024] env[63593]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 898.489024] env[63593]: nwinfo = self.network_api.allocate_for_instance( [ 898.489024] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 898.489024] env[63593]: created_port_ids = self._update_ports_for_instance( [ 898.489024] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 898.489024] env[63593]: with excutils.save_and_reraise_exception(): [ 898.489024] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 898.489024] env[63593]: self.force_reraise() [ 898.489024] env[63593]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 898.489024] env[63593]: raise self.value [ 898.489024] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 898.489024] env[63593]: updated_port = self._update_port( [ 898.489024] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 898.489024] env[63593]: _ensure_no_port_binding_failure(port) [ 898.489024] env[63593]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 898.489024] env[63593]: raise exception.PortBindingFailed(port_id=port['id']) [ 898.489804] env[63593]: nova.exception.PortBindingFailed: Binding failed for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11, please check neutron logs for more information. [ 898.489804] env[63593]: Removing descriptor: 18 [ 898.519039] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 898.595081] env[63593]: WARNING oslo_vmware.rw_handles [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 898.595081] env[63593]: ERROR oslo_vmware.rw_handles [ 898.595081] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/bba2d177-d1ca-4c03-82b0-c47a80fb4f61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 898.597789] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 898.597789] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Copying Virtual Disk [datastore2] vmware_temp/bba2d177-d1ca-4c03-82b0-c47a80fb4f61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/bba2d177-d1ca-4c03-82b0-c47a80fb4f61/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 898.597990] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-454a3ca1-fd4b-44fd-8b4d-ef74a9e15121 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.609475] env[63593]: DEBUG oslo_vmware.api [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for the task: (returnval){ [ 898.609475] env[63593]: value = "task-1367968" [ 898.609475] env[63593]: _type = "Task" [ 898.609475] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 898.617592] env[63593]: DEBUG oslo_vmware.api [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': task-1367968, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 898.641039] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 898.667986] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 898.668259] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 898.668415] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 898.668655] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 898.668861] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 898.669019] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 898.669229] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 898.669386] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 898.669549] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 898.669711] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 898.669916] env[63593]: DEBUG nova.virt.hardware [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 898.670717] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d45b97e-721e-4376-970e-8ea602404cf7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.678146] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-744e175b-7740-48f4-b7f3-43e9cd1078b4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.691705] env[63593]: ERROR nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11, please check neutron logs for more information. [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Traceback (most recent call last): [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] yield resources [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self.driver.spawn(context, instance, image_meta, [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self._vmops.spawn(context, instance, image_meta, injected_files, [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] vm_ref = self.build_virtual_machine(instance, [ 898.691705] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] vif_infos = vmwarevif.get_vif_info(self._session, [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] for vif in network_info: [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] return self._sync_wrapper(fn, *args, **kwargs) [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self.wait() [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self[:] = self._gt.wait() [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] return self._exit_event.wait() [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 898.692083] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] current.throw(*self._exc) [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] result = function(*args, **kwargs) [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] return func(*args, **kwargs) [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] raise e [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] nwinfo = self.network_api.allocate_for_instance( [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] created_port_ids = self._update_ports_for_instance( [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] with excutils.save_and_reraise_exception(): [ 898.692366] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self.force_reraise() [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] raise self.value [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] updated_port = self._update_port( [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] _ensure_no_port_binding_failure(port) [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] raise exception.PortBindingFailed(port_id=port['id']) [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] nova.exception.PortBindingFailed: Binding failed for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11, please check neutron logs for more information. [ 898.692650] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] [ 898.692650] env[63593]: INFO nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Terminating instance [ 898.694023] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Acquiring lock "refresh_cache-ef8646c6-7ead-4c9f-8e8f-85c758a35006" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 898.801848] env[63593]: DEBUG nova.network.neutron [req-50763e42-3f24-416d-a8ac-d577f3155138 req-7e04bf95-4156-46dc-964a-04266cae6c88 service nova] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 898.896909] env[63593]: DEBUG nova.network.neutron [req-50763e42-3f24-416d-a8ac-d577f3155138 req-7e04bf95-4156-46dc-964a-04266cae6c88 service nova] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 898.986370] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.376s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.987172] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 898.989943] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 22.138s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 898.991370] env[63593]: INFO nova.compute.claims [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 899.049310] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 899.120807] env[63593]: DEBUG oslo_vmware.exceptions [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 899.121174] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 899.121783] env[63593]: ERROR nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 899.121783] env[63593]: Faults: ['InvalidArgument'] [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Traceback (most recent call last): [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] yield resources [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] self.driver.spawn(context, instance, image_meta, [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] self._vmops.spawn(context, instance, image_meta, injected_files, [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] self._fetch_image_if_missing(context, vi) [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 899.121783] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] image_cache(vi, tmp_image_ds_loc) [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] vm_util.copy_virtual_disk( [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] session._wait_for_task(vmdk_copy_task) [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] return self.wait_for_task(task_ref) [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] return evt.wait() [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] result = hub.switch() [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] return self.greenlet.switch() [ 899.122118] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 899.122468] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] self.f(*self.args, **self.kw) [ 899.122468] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 899.122468] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] raise exceptions.translate_fault(task_info.error) [ 899.122468] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 899.122468] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Faults: ['InvalidArgument'] [ 899.122468] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] [ 899.122468] env[63593]: INFO nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Terminating instance [ 899.123594] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 899.123800] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 899.125530] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "refresh_cache-94ff0182-d1ce-405f-bdb5-a2399bd0f4de" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 899.125530] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired lock "refresh_cache-94ff0182-d1ce-405f-bdb5-a2399bd0f4de" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 899.125530] env[63593]: DEBUG nova.network.neutron [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 899.125761] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e1c359a5-81b9-4db1-ac79-993e32513ae2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.134455] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 899.134455] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 899.135237] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-bfa972e8-ba41-4e97-9262-d6f02230fbe4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.140420] env[63593]: DEBUG oslo_vmware.api [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for the task: (returnval){ [ 899.140420] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]523e9bb8-4772-44e4-6cf9-d890c3c06c6f" [ 899.140420] env[63593]: _type = "Task" [ 899.140420] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 899.147988] env[63593]: DEBUG oslo_vmware.api [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]523e9bb8-4772-44e4-6cf9-d890c3c06c6f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 899.399470] env[63593]: DEBUG oslo_concurrency.lockutils [req-50763e42-3f24-416d-a8ac-d577f3155138 req-7e04bf95-4156-46dc-964a-04266cae6c88 service nova] Releasing lock "refresh_cache-ef8646c6-7ead-4c9f-8e8f-85c758a35006" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 899.400951] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Acquired lock "refresh_cache-ef8646c6-7ead-4c9f-8e8f-85c758a35006" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 899.400951] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 899.495692] env[63593]: DEBUG nova.compute.utils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 899.499026] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 899.499026] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 899.542109] env[63593]: DEBUG nova.policy [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea62c292bb2b4922a560a64191173304', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '83b28b01efea4d77b238410c62cea17d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 899.654906] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 899.655278] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Creating directory with path [datastore2] vmware_temp/10abfe60-f21d-49b2-ab2b-822ce0f0d754/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 899.655571] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-46cb254c-51e9-4bf4-a664-9a933d74355b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.658143] env[63593]: DEBUG nova.network.neutron [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 899.670017] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Created directory with path [datastore2] vmware_temp/10abfe60-f21d-49b2-ab2b-822ce0f0d754/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 899.670317] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Fetch image to [datastore2] vmware_temp/10abfe60-f21d-49b2-ab2b-822ce0f0d754/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 899.670546] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/10abfe60-f21d-49b2-ab2b-822ce0f0d754/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 899.671335] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d763a2f5-d361-48c1-acbe-9f0b502dceb0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.677948] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35c75432-6b58-4437-b604-79b6824bb03a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.689958] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e324951f-8d2f-4236-8af3-119634304fd4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.719404] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-651a2f53-f088-4a3a-aab5-6c4649e613a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.725178] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-256471b7-c42b-4566-9e1e-6e2ab2d40898 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.742201] env[63593]: DEBUG nova.network.neutron [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 899.745783] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 899.827684] env[63593]: DEBUG oslo_vmware.rw_handles [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/10abfe60-f21d-49b2-ab2b-822ce0f0d754/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 899.829450] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Successfully created port: 585f4df1-77e8-4402-8082-6d9f3fc42abb {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 899.893377] env[63593]: DEBUG oslo_vmware.rw_handles [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 899.893688] env[63593]: DEBUG oslo_vmware.rw_handles [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/10abfe60-f21d-49b2-ab2b-822ce0f0d754/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 899.918523] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 899.988599] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 899.999413] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 900.249524] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Releasing lock "refresh_cache-94ff0182-d1ce-405f-bdb5-a2399bd0f4de" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 900.249922] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 900.250138] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 900.250985] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ca0b859-20d2-4663-8311-083f3fe24149 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.262069] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 900.262307] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-00cbe338-731c-486a-b5d2-a4fcce74b36a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.304084] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 900.304183] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 900.304338] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Deleting the datastore file [datastore2] 94ff0182-d1ce-405f-bdb5-a2399bd0f4de {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 900.304945] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-601d1c73-464e-44b8-8e0c-4d0659b44030 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.314552] env[63593]: DEBUG oslo_vmware.api [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for the task: (returnval){ [ 900.314552] env[63593]: value = "task-1367970" [ 900.314552] env[63593]: _type = "Task" [ 900.314552] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 900.323432] env[63593]: DEBUG oslo_vmware.api [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': task-1367970, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 900.339050] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-657b1701-00c3-4e1d-8857-0fed39c10fbe {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.345971] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a4521b4-f64c-449f-b386-4ac5d4101144 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.380881] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85c36693-99ab-43f7-b649-e0575d342873 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.386952] env[63593]: DEBUG nova.compute.manager [req-abcafe3a-0347-45f3-834a-01a5b4e82323 req-e307ecb2-2a63-4ece-9b56-80f77adfda1e service nova] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Received event network-vif-deleted-fdb0f1c9-485b-4b15-94d3-e7d274b52f11 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 900.392124] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c524af02-264b-4bfb-96bf-f558b49b62e1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.408975] env[63593]: DEBUG nova.compute.provider_tree [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 900.491421] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Releasing lock "refresh_cache-ef8646c6-7ead-4c9f-8e8f-85c758a35006" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 900.492518] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 900.492518] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 900.492518] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d83f66b0-7336-4e6a-a7b5-83b0f05bb8db {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.502961] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e908ed1-fb7a-4145-a75b-bab40273313f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.532567] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ef8646c6-7ead-4c9f-8e8f-85c758a35006 could not be found. [ 900.532969] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 900.532969] env[63593]: INFO nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Took 0.04 seconds to destroy the instance on the hypervisor. [ 900.533201] env[63593]: DEBUG oslo.service.loopingcall [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 900.533428] env[63593]: DEBUG nova.compute.manager [-] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 900.533523] env[63593]: DEBUG nova.network.neutron [-] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 900.551615] env[63593]: DEBUG nova.network.neutron [-] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 900.824157] env[63593]: DEBUG oslo_vmware.api [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': task-1367970, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.037486} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 900.824446] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 900.825027] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 900.825027] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 900.825027] env[63593]: INFO nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Took 0.57 seconds to destroy the instance on the hypervisor. [ 900.825175] env[63593]: DEBUG oslo.service.loopingcall [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 900.825576] env[63593]: DEBUG nova.compute.manager [-] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 900.829820] env[63593]: DEBUG nova.compute.claims [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 900.830065] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 900.913751] env[63593]: DEBUG nova.scheduler.client.report [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 901.021676] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 901.047422] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 901.047673] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 901.047830] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 901.048015] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 901.048176] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 901.048322] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 901.048525] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 901.048712] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 901.048886] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 901.049059] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 901.049232] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 901.050159] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e57356f5-2779-49cc-b8b5-fb336f1c6655 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.054178] env[63593]: DEBUG nova.network.neutron [-] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 901.059364] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd55ac71-1cfc-40c2-bb5f-fc742b9e0af8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.418277] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.428s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 901.419167] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 901.422274] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.062s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 901.424444] env[63593]: INFO nova.compute.claims [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 901.556704] env[63593]: INFO nova.compute.manager [-] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Took 1.02 seconds to deallocate network for instance. [ 901.559173] env[63593]: DEBUG nova.compute.claims [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 901.559358] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 901.767737] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Successfully updated port: 585f4df1-77e8-4402-8082-6d9f3fc42abb {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 901.929733] env[63593]: DEBUG nova.compute.utils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 901.933377] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 901.933592] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 901.970802] env[63593]: DEBUG nova.policy [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea62c292bb2b4922a560a64191173304', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '83b28b01efea4d77b238410c62cea17d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 902.237648] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Successfully created port: 61f5c912-202c-47bb-8425-9cccc65553c6 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 902.270550] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "refresh_cache-a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 902.270705] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "refresh_cache-a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 902.270858] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 902.390064] env[63593]: DEBUG nova.compute.manager [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Received event network-vif-plugged-585f4df1-77e8-4402-8082-6d9f3fc42abb {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 902.390323] env[63593]: DEBUG oslo_concurrency.lockutils [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] Acquiring lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 902.390831] env[63593]: DEBUG oslo_concurrency.lockutils [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] Lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 902.391214] env[63593]: DEBUG oslo_concurrency.lockutils [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] Lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 902.391483] env[63593]: DEBUG nova.compute.manager [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] No waiting events found dispatching network-vif-plugged-585f4df1-77e8-4402-8082-6d9f3fc42abb {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 902.391793] env[63593]: WARNING nova.compute.manager [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Received unexpected event network-vif-plugged-585f4df1-77e8-4402-8082-6d9f3fc42abb for instance with vm_state building and task_state spawning. [ 902.392186] env[63593]: DEBUG nova.compute.manager [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Received event network-changed-585f4df1-77e8-4402-8082-6d9f3fc42abb {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 902.392373] env[63593]: DEBUG nova.compute.manager [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Refreshing instance network info cache due to event network-changed-585f4df1-77e8-4402-8082-6d9f3fc42abb. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 902.392662] env[63593]: DEBUG oslo_concurrency.lockutils [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] Acquiring lock "refresh_cache-a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 902.434603] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 902.735259] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99687d9f-ff92-44fb-86a1-5bdfe1001815 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.742614] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa96facd-6d84-49f7-b648-b12d27c07126 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.771876] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac82a692-1df7-4c2d-a473-34df9d9e2e66 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.781033] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91462a79-e690-48fd-bfb2-c2d03dbc427a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.793433] env[63593]: DEBUG nova.compute.provider_tree [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 902.804925] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 903.140363] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Updating instance_info_cache with network_info: [{"id": "585f4df1-77e8-4402-8082-6d9f3fc42abb", "address": "fa:16:3e:61:2e:27", "network": {"id": "fc892db8-160b-493a-87c1-fb7eebc2d6ee", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-273487004-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "83b28b01efea4d77b238410c62cea17d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d4b43a78-f49b-4132-ab2e-6e28769a9498", "external-id": "nsx-vlan-transportzone-737", "segmentation_id": 737, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap585f4df1-77", "ovs_interfaceid": "585f4df1-77e8-4402-8082-6d9f3fc42abb", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 903.298041] env[63593]: DEBUG nova.scheduler.client.report [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 903.446877] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 903.474136] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 903.474488] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 903.474712] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 903.474966] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 903.475187] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 903.475396] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 903.475684] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 903.475904] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 903.476152] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 903.476379] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 903.476610] env[63593]: DEBUG nova.virt.hardware [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 903.477810] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c16494c9-f130-48b3-a622-3c65280c330f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.489960] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a7e51d9-a0c6-4964-b9f8-680ff8bf6e5f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.643541] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "refresh_cache-a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 903.643889] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Instance network_info: |[{"id": "585f4df1-77e8-4402-8082-6d9f3fc42abb", "address": "fa:16:3e:61:2e:27", "network": {"id": "fc892db8-160b-493a-87c1-fb7eebc2d6ee", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-273487004-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "83b28b01efea4d77b238410c62cea17d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d4b43a78-f49b-4132-ab2e-6e28769a9498", "external-id": "nsx-vlan-transportzone-737", "segmentation_id": 737, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap585f4df1-77", "ovs_interfaceid": "585f4df1-77e8-4402-8082-6d9f3fc42abb", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 903.644260] env[63593]: DEBUG oslo_concurrency.lockutils [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] Acquired lock "refresh_cache-a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 903.644438] env[63593]: DEBUG nova.network.neutron [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Refreshing network info cache for port 585f4df1-77e8-4402-8082-6d9f3fc42abb {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 903.645869] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:61:2e:27', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'd4b43a78-f49b-4132-ab2e-6e28769a9498', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '585f4df1-77e8-4402-8082-6d9f3fc42abb', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 903.653998] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Creating folder: Project (83b28b01efea4d77b238410c62cea17d). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 903.655119] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-8d5d2327-7919-4369-bf96-f843c0ec80d1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.666827] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Created folder: Project (83b28b01efea4d77b238410c62cea17d) in parent group-v291016. [ 903.667043] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Creating folder: Instances. Parent ref: group-v291038. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 903.667281] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-76aa8aec-852a-479e-ac7e-5de2e394431b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.675646] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Created folder: Instances in parent group-v291038. [ 903.675876] env[63593]: DEBUG oslo.service.loopingcall [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 903.676073] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 903.676267] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-244e0ad4-9d7a-4caa-b4e3-a8af88bf3352 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.695176] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 903.695176] env[63593]: value = "task-1367973" [ 903.695176] env[63593]: _type = "Task" [ 903.695176] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 903.702422] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367973, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 903.777442] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Successfully updated port: 61f5c912-202c-47bb-8425-9cccc65553c6 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 903.801229] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.379s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 903.801812] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 903.804654] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.517s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 903.806034] env[63593]: INFO nova.compute.claims [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 904.205796] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367973, 'name': CreateVM_Task, 'duration_secs': 0.288817} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 904.205962] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 904.213126] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 904.213304] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 904.213635] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 904.213866] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-88227fdb-16fc-41b1-aab9-2a6702d5a811 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.218088] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for the task: (returnval){ [ 904.218088] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]524b9433-d370-daf8-0240-7ed84b7efa6e" [ 904.218088] env[63593]: _type = "Task" [ 904.218088] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 904.225225] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]524b9433-d370-daf8-0240-7ed84b7efa6e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 904.279965] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "refresh_cache-d3a4902f-1829-427c-9f37-6e83ae7265e3" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 904.280121] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "refresh_cache-d3a4902f-1829-427c-9f37-6e83ae7265e3" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 904.280275] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 904.311025] env[63593]: DEBUG nova.compute.utils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 904.317274] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 904.317455] env[63593]: DEBUG nova.network.neutron [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 904.366947] env[63593]: DEBUG nova.policy [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b266e5f220334e3ab738bba46c0b12d9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'baf0ac930fbc46d49ae28c0abb4815cd', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 904.417024] env[63593]: DEBUG nova.network.neutron [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Updated VIF entry in instance network info cache for port 585f4df1-77e8-4402-8082-6d9f3fc42abb. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 904.417398] env[63593]: DEBUG nova.network.neutron [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Updating instance_info_cache with network_info: [{"id": "585f4df1-77e8-4402-8082-6d9f3fc42abb", "address": "fa:16:3e:61:2e:27", "network": {"id": "fc892db8-160b-493a-87c1-fb7eebc2d6ee", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-273487004-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "83b28b01efea4d77b238410c62cea17d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d4b43a78-f49b-4132-ab2e-6e28769a9498", "external-id": "nsx-vlan-transportzone-737", "segmentation_id": 737, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap585f4df1-77", "ovs_interfaceid": "585f4df1-77e8-4402-8082-6d9f3fc42abb", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 904.421793] env[63593]: DEBUG nova.compute.manager [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Received event network-vif-plugged-61f5c912-202c-47bb-8425-9cccc65553c6 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 904.421910] env[63593]: DEBUG oslo_concurrency.lockutils [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] Acquiring lock "d3a4902f-1829-427c-9f37-6e83ae7265e3-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 904.422170] env[63593]: DEBUG oslo_concurrency.lockutils [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] Lock "d3a4902f-1829-427c-9f37-6e83ae7265e3-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 904.422283] env[63593]: DEBUG oslo_concurrency.lockutils [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] Lock "d3a4902f-1829-427c-9f37-6e83ae7265e3-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 904.422447] env[63593]: DEBUG nova.compute.manager [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] No waiting events found dispatching network-vif-plugged-61f5c912-202c-47bb-8425-9cccc65553c6 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 904.422607] env[63593]: WARNING nova.compute.manager [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Received unexpected event network-vif-plugged-61f5c912-202c-47bb-8425-9cccc65553c6 for instance with vm_state building and task_state spawning. [ 904.422762] env[63593]: DEBUG nova.compute.manager [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Received event network-changed-61f5c912-202c-47bb-8425-9cccc65553c6 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 904.422912] env[63593]: DEBUG nova.compute.manager [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Refreshing instance network info cache due to event network-changed-61f5c912-202c-47bb-8425-9cccc65553c6. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 904.423082] env[63593]: DEBUG oslo_concurrency.lockutils [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] Acquiring lock "refresh_cache-d3a4902f-1829-427c-9f37-6e83ae7265e3" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 904.624419] env[63593]: DEBUG nova.network.neutron [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Successfully created port: 015d8af3-1e13-4cd3-8873-1f277b7566dd {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 904.729722] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 904.731391] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 904.731391] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 904.813477] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 904.825064] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 904.919823] env[63593]: DEBUG oslo_concurrency.lockutils [req-67c9ec2e-ba6a-4f0f-adc3-58c22ac5151a req-5aa83172-9fef-4048-881c-cd0ad9878cff service nova] Releasing lock "refresh_cache-a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 905.014367] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Updating instance_info_cache with network_info: [{"id": "61f5c912-202c-47bb-8425-9cccc65553c6", "address": "fa:16:3e:eb:54:02", "network": {"id": "fc892db8-160b-493a-87c1-fb7eebc2d6ee", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-273487004-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "83b28b01efea4d77b238410c62cea17d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d4b43a78-f49b-4132-ab2e-6e28769a9498", "external-id": "nsx-vlan-transportzone-737", "segmentation_id": 737, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap61f5c912-20", "ovs_interfaceid": "61f5c912-202c-47bb-8425-9cccc65553c6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 905.104158] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-446b72c3-c313-46d8-98f9-be490f67643f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.109799] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c1ae1fe-e589-460a-a022-6a4f63fc06ed {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.140814] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9f3f88b-aac5-42a0-8776-0c7f685051d0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.147765] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79238920-a4df-4063-949a-8f74cd88f6a6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.161842] env[63593]: DEBUG nova.compute.provider_tree [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 905.518853] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "refresh_cache-d3a4902f-1829-427c-9f37-6e83ae7265e3" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 905.519252] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Instance network_info: |[{"id": "61f5c912-202c-47bb-8425-9cccc65553c6", "address": "fa:16:3e:eb:54:02", "network": {"id": "fc892db8-160b-493a-87c1-fb7eebc2d6ee", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-273487004-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "83b28b01efea4d77b238410c62cea17d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d4b43a78-f49b-4132-ab2e-6e28769a9498", "external-id": "nsx-vlan-transportzone-737", "segmentation_id": 737, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap61f5c912-20", "ovs_interfaceid": "61f5c912-202c-47bb-8425-9cccc65553c6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 905.519559] env[63593]: DEBUG oslo_concurrency.lockutils [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] Acquired lock "refresh_cache-d3a4902f-1829-427c-9f37-6e83ae7265e3" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 905.519738] env[63593]: DEBUG nova.network.neutron [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Refreshing network info cache for port 61f5c912-202c-47bb-8425-9cccc65553c6 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 905.520921] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:eb:54:02', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'd4b43a78-f49b-4132-ab2e-6e28769a9498', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '61f5c912-202c-47bb-8425-9cccc65553c6', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 905.528246] env[63593]: DEBUG oslo.service.loopingcall [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 905.529184] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 905.529404] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3add4a9d-e4be-4e58-b69e-1b438d3e2dcf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.549568] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 905.549568] env[63593]: value = "task-1367974" [ 905.549568] env[63593]: _type = "Task" [ 905.549568] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 905.558647] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367974, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 905.664589] env[63593]: DEBUG nova.scheduler.client.report [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 905.823368] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 905.849523] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 905.849766] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 905.849948] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 905.850152] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 905.850290] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 905.850431] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 905.850631] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 905.850785] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 905.850950] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 905.851130] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 905.851300] env[63593]: DEBUG nova.virt.hardware [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 905.852172] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b135cd2-d012-4d98-9856-93b83d912b7e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.860079] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38f1c179-cf77-4c01-be15-a1baf1194959 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.058729] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367974, 'name': CreateVM_Task, 'duration_secs': 0.303737} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 906.059128] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 906.062017] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 906.062017] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 906.062017] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 906.062017] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-19ef112b-71d0-4bf7-bb68-ce76366e0c22 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.065295] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for the task: (returnval){ [ 906.065295] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52db4ef6-eee3-f60a-fd35-27423d74099f" [ 906.065295] env[63593]: _type = "Task" [ 906.065295] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 906.080667] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52db4ef6-eee3-f60a-fd35-27423d74099f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 906.169938] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.365s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.170510] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 906.173268] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.873s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.174759] env[63593]: INFO nova.compute.claims [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 906.190320] env[63593]: DEBUG nova.compute.manager [req-4f9569cd-3e30-49ca-af35-63129d57ef57 req-579724c2-6f5b-4bdc-9b0c-d78780e1aac2 service nova] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Received event network-vif-plugged-015d8af3-1e13-4cd3-8873-1f277b7566dd {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 906.190554] env[63593]: DEBUG oslo_concurrency.lockutils [req-4f9569cd-3e30-49ca-af35-63129d57ef57 req-579724c2-6f5b-4bdc-9b0c-d78780e1aac2 service nova] Acquiring lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 906.190813] env[63593]: DEBUG oslo_concurrency.lockutils [req-4f9569cd-3e30-49ca-af35-63129d57ef57 req-579724c2-6f5b-4bdc-9b0c-d78780e1aac2 service nova] Lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.190933] env[63593]: DEBUG oslo_concurrency.lockutils [req-4f9569cd-3e30-49ca-af35-63129d57ef57 req-579724c2-6f5b-4bdc-9b0c-d78780e1aac2 service nova] Lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.191115] env[63593]: DEBUG nova.compute.manager [req-4f9569cd-3e30-49ca-af35-63129d57ef57 req-579724c2-6f5b-4bdc-9b0c-d78780e1aac2 service nova] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] No waiting events found dispatching network-vif-plugged-015d8af3-1e13-4cd3-8873-1f277b7566dd {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 906.191277] env[63593]: WARNING nova.compute.manager [req-4f9569cd-3e30-49ca-af35-63129d57ef57 req-579724c2-6f5b-4bdc-9b0c-d78780e1aac2 service nova] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Received unexpected event network-vif-plugged-015d8af3-1e13-4cd3-8873-1f277b7566dd for instance with vm_state building and task_state spawning. [ 906.288895] env[63593]: DEBUG nova.network.neutron [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Successfully updated port: 015d8af3-1e13-4cd3-8873-1f277b7566dd {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 906.299677] env[63593]: DEBUG nova.network.neutron [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Updated VIF entry in instance network info cache for port 61f5c912-202c-47bb-8425-9cccc65553c6. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 906.300170] env[63593]: DEBUG nova.network.neutron [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Updating instance_info_cache with network_info: [{"id": "61f5c912-202c-47bb-8425-9cccc65553c6", "address": "fa:16:3e:eb:54:02", "network": {"id": "fc892db8-160b-493a-87c1-fb7eebc2d6ee", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-273487004-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "83b28b01efea4d77b238410c62cea17d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d4b43a78-f49b-4132-ab2e-6e28769a9498", "external-id": "nsx-vlan-transportzone-737", "segmentation_id": 737, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap61f5c912-20", "ovs_interfaceid": "61f5c912-202c-47bb-8425-9cccc65553c6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 906.576458] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 906.576734] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 906.576961] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 906.679776] env[63593]: DEBUG nova.compute.utils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 906.684455] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 906.684639] env[63593]: DEBUG nova.network.neutron [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 906.724163] env[63593]: DEBUG nova.policy [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2a9e98a21f854388b8ed32d3662263bf', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2f95123fa26d4061b5748fdb689e5985', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 906.791471] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquiring lock "refresh_cache-efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 906.791623] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquired lock "refresh_cache-efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 906.791774] env[63593]: DEBUG nova.network.neutron [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 906.803419] env[63593]: DEBUG oslo_concurrency.lockutils [req-c4cd2408-4aff-455d-8890-9222cc801f28 req-17fbf93d-762a-4605-90a2-eedf969099f6 service nova] Releasing lock "refresh_cache-d3a4902f-1829-427c-9f37-6e83ae7265e3" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 906.982605] env[63593]: DEBUG nova.network.neutron [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Successfully created port: c3739fab-ec53-42cb-82cc-dd4c699fd80b {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 907.185078] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 907.327934] env[63593]: DEBUG nova.network.neutron [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 907.460819] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0606f685-aa66-456a-8475-8191dcb5cc7b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.468978] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b56ad2c-5d08-45fc-ba0a-2c63af8c5e2d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.505956] env[63593]: DEBUG nova.network.neutron [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Updating instance_info_cache with network_info: [{"id": "015d8af3-1e13-4cd3-8873-1f277b7566dd", "address": "fa:16:3e:1e:40:95", "network": {"id": "172012e7-297f-451d-80b9-fe1fb3384edc", "bridge": "br-int", "label": "tempest-ServerAddressesNegativeTestJSON-1680621240-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "baf0ac930fbc46d49ae28c0abb4815cd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "2c2daf7c-c01b-41b1-a09a-fb8b893b4c80", "external-id": "nsx-vlan-transportzone-89", "segmentation_id": 89, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap015d8af3-1e", "ovs_interfaceid": "015d8af3-1e13-4cd3-8873-1f277b7566dd", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 907.508983] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ca9017c-ec21-4b2a-8341-5a5bca9a40a1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.518466] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ae98320-7b73-470d-9bb0-ed498dd8602c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.535882] env[63593]: DEBUG nova.compute.provider_tree [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 908.013540] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Releasing lock "refresh_cache-efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 908.013912] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Instance network_info: |[{"id": "015d8af3-1e13-4cd3-8873-1f277b7566dd", "address": "fa:16:3e:1e:40:95", "network": {"id": "172012e7-297f-451d-80b9-fe1fb3384edc", "bridge": "br-int", "label": "tempest-ServerAddressesNegativeTestJSON-1680621240-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "baf0ac930fbc46d49ae28c0abb4815cd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "2c2daf7c-c01b-41b1-a09a-fb8b893b4c80", "external-id": "nsx-vlan-transportzone-89", "segmentation_id": 89, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap015d8af3-1e", "ovs_interfaceid": "015d8af3-1e13-4cd3-8873-1f277b7566dd", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 908.014328] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:1e:40:95', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '2c2daf7c-c01b-41b1-a09a-fb8b893b4c80', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '015d8af3-1e13-4cd3-8873-1f277b7566dd', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 908.021955] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Creating folder: Project (baf0ac930fbc46d49ae28c0abb4815cd). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 908.022233] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b0b78d43-2523-4e3d-97fd-043d57268df8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.034683] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Created folder: Project (baf0ac930fbc46d49ae28c0abb4815cd) in parent group-v291016. [ 908.034861] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Creating folder: Instances. Parent ref: group-v291042. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 908.035081] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0e5c90fd-9eb4-448e-9d3d-97b628dd405e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.039615] env[63593]: DEBUG nova.scheduler.client.report [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 908.043848] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Created folder: Instances in parent group-v291042. [ 908.043965] env[63593]: DEBUG oslo.service.loopingcall [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 908.044374] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 908.044568] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-8d171a36-0ca3-4f63-bd97-98c3613ff288 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.064559] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 908.064559] env[63593]: value = "task-1367977" [ 908.064559] env[63593]: _type = "Task" [ 908.064559] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 908.073963] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367977, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 908.208117] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 908.217886] env[63593]: DEBUG nova.compute.manager [req-1b8157ac-e73d-41e7-9f43-61295cf8ace9 req-674bf91f-0322-4ec6-8394-7af8a5fd82b6 service nova] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Received event network-changed-015d8af3-1e13-4cd3-8873-1f277b7566dd {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 908.218311] env[63593]: DEBUG nova.compute.manager [req-1b8157ac-e73d-41e7-9f43-61295cf8ace9 req-674bf91f-0322-4ec6-8394-7af8a5fd82b6 service nova] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Refreshing instance network info cache due to event network-changed-015d8af3-1e13-4cd3-8873-1f277b7566dd. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 908.218670] env[63593]: DEBUG oslo_concurrency.lockutils [req-1b8157ac-e73d-41e7-9f43-61295cf8ace9 req-674bf91f-0322-4ec6-8394-7af8a5fd82b6 service nova] Acquiring lock "refresh_cache-efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 908.218978] env[63593]: DEBUG oslo_concurrency.lockutils [req-1b8157ac-e73d-41e7-9f43-61295cf8ace9 req-674bf91f-0322-4ec6-8394-7af8a5fd82b6 service nova] Acquired lock "refresh_cache-efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 908.219270] env[63593]: DEBUG nova.network.neutron [req-1b8157ac-e73d-41e7-9f43-61295cf8ace9 req-674bf91f-0322-4ec6-8394-7af8a5fd82b6 service nova] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Refreshing network info cache for port 015d8af3-1e13-4cd3-8873-1f277b7566dd {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 908.233798] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 908.234210] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 908.234316] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 908.234530] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 908.234677] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 908.234823] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 908.235076] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 908.235255] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 908.235423] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 908.235583] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 908.235753] env[63593]: DEBUG nova.virt.hardware [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 908.236846] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b123932f-152c-423e-aa70-563a40e159a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.245248] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5814e8d-7a50-436f-b6ac-7ab288b125be {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.549062] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.372s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 908.549062] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 908.555179] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.504s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 908.557262] env[63593]: INFO nova.compute.claims [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 908.561160] env[63593]: DEBUG nova.compute.manager [req-995504d5-9db9-429c-9052-ab75108e317a req-1e2e6609-4bc9-4e56-aa08-44c95fb7125f service nova] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Received event network-vif-plugged-c3739fab-ec53-42cb-82cc-dd4c699fd80b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 908.561487] env[63593]: DEBUG oslo_concurrency.lockutils [req-995504d5-9db9-429c-9052-ab75108e317a req-1e2e6609-4bc9-4e56-aa08-44c95fb7125f service nova] Acquiring lock "fbd620a6-3743-4737-98b7-b13928b6587d-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 908.561799] env[63593]: DEBUG oslo_concurrency.lockutils [req-995504d5-9db9-429c-9052-ab75108e317a req-1e2e6609-4bc9-4e56-aa08-44c95fb7125f service nova] Lock "fbd620a6-3743-4737-98b7-b13928b6587d-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 908.562078] env[63593]: DEBUG oslo_concurrency.lockutils [req-995504d5-9db9-429c-9052-ab75108e317a req-1e2e6609-4bc9-4e56-aa08-44c95fb7125f service nova] Lock "fbd620a6-3743-4737-98b7-b13928b6587d-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 908.562348] env[63593]: DEBUG nova.compute.manager [req-995504d5-9db9-429c-9052-ab75108e317a req-1e2e6609-4bc9-4e56-aa08-44c95fb7125f service nova] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] No waiting events found dispatching network-vif-plugged-c3739fab-ec53-42cb-82cc-dd4c699fd80b {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 908.562613] env[63593]: WARNING nova.compute.manager [req-995504d5-9db9-429c-9052-ab75108e317a req-1e2e6609-4bc9-4e56-aa08-44c95fb7125f service nova] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Received unexpected event network-vif-plugged-c3739fab-ec53-42cb-82cc-dd4c699fd80b for instance with vm_state building and task_state spawning. [ 908.575830] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367977, 'name': CreateVM_Task, 'duration_secs': 0.309267} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 908.575830] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 908.576157] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 908.576522] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 908.576953] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 908.577304] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-26b3b306-ed80-442c-96f5-3003c6276117 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.582505] env[63593]: DEBUG oslo_vmware.api [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Waiting for the task: (returnval){ [ 908.582505] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52b4eb7b-c675-57e7-40cc-087b6fdace27" [ 908.582505] env[63593]: _type = "Task" [ 908.582505] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 908.590344] env[63593]: DEBUG oslo_vmware.api [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52b4eb7b-c675-57e7-40cc-087b6fdace27, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 908.645035] env[63593]: DEBUG nova.network.neutron [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Successfully updated port: c3739fab-ec53-42cb-82cc-dd4c699fd80b {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 908.923659] env[63593]: DEBUG nova.network.neutron [req-1b8157ac-e73d-41e7-9f43-61295cf8ace9 req-674bf91f-0322-4ec6-8394-7af8a5fd82b6 service nova] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Updated VIF entry in instance network info cache for port 015d8af3-1e13-4cd3-8873-1f277b7566dd. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 908.924045] env[63593]: DEBUG nova.network.neutron [req-1b8157ac-e73d-41e7-9f43-61295cf8ace9 req-674bf91f-0322-4ec6-8394-7af8a5fd82b6 service nova] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Updating instance_info_cache with network_info: [{"id": "015d8af3-1e13-4cd3-8873-1f277b7566dd", "address": "fa:16:3e:1e:40:95", "network": {"id": "172012e7-297f-451d-80b9-fe1fb3384edc", "bridge": "br-int", "label": "tempest-ServerAddressesNegativeTestJSON-1680621240-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "baf0ac930fbc46d49ae28c0abb4815cd", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "2c2daf7c-c01b-41b1-a09a-fb8b893b4c80", "external-id": "nsx-vlan-transportzone-89", "segmentation_id": 89, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap015d8af3-1e", "ovs_interfaceid": "015d8af3-1e13-4cd3-8873-1f277b7566dd", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 909.069076] env[63593]: DEBUG nova.compute.utils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 909.069781] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 909.069964] env[63593]: DEBUG nova.network.neutron [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 909.093059] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 909.093309] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 909.093521] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 909.108554] env[63593]: DEBUG nova.policy [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a716a2c8efe046e9923bcd6b7956b22a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '60d47be26aac44d198ca43a997485f7c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 909.147088] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "refresh_cache-fbd620a6-3743-4737-98b7-b13928b6587d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 909.147221] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquired lock "refresh_cache-fbd620a6-3743-4737-98b7-b13928b6587d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 909.147372] env[63593]: DEBUG nova.network.neutron [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 909.412505] env[63593]: DEBUG nova.network.neutron [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Successfully created port: a1f7f8bf-af7d-4296-b234-000d10f51b2b {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 909.426986] env[63593]: DEBUG oslo_concurrency.lockutils [req-1b8157ac-e73d-41e7-9f43-61295cf8ace9 req-674bf91f-0322-4ec6-8394-7af8a5fd82b6 service nova] Releasing lock "refresh_cache-efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 909.574674] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 909.682580] env[63593]: DEBUG nova.network.neutron [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 909.821763] env[63593]: DEBUG nova.network.neutron [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Updating instance_info_cache with network_info: [{"id": "c3739fab-ec53-42cb-82cc-dd4c699fd80b", "address": "fa:16:3e:b9:8c:30", "network": {"id": "77bd00f0-884e-4553-b133-ebe7f63c5d20", "bridge": "br-int", "label": "tempest-VolumesAdminNegativeTest-537197341-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "2f95123fa26d4061b5748fdb689e5985", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "980cb890-345b-4bf8-990a-a2faec78e49c", "external-id": "nsx-vlan-transportzone-965", "segmentation_id": 965, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc3739fab-ec", "ovs_interfaceid": "c3739fab-ec53-42cb-82cc-dd4c699fd80b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 909.856500] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f56fa657-6020-40d4-a376-28ff33906ee7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.864183] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9da29904-cfaa-4676-a6dc-47454be8798d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.894036] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df1d0ab0-e784-4d31-8d5c-a5e440e370e1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.901103] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09d91d51-e657-4a96-bb26-594481632dbc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 909.913893] env[63593]: DEBUG nova.compute.provider_tree [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 910.324954] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Releasing lock "refresh_cache-fbd620a6-3743-4737-98b7-b13928b6587d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 910.325500] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Instance network_info: |[{"id": "c3739fab-ec53-42cb-82cc-dd4c699fd80b", "address": "fa:16:3e:b9:8c:30", "network": {"id": "77bd00f0-884e-4553-b133-ebe7f63c5d20", "bridge": "br-int", "label": "tempest-VolumesAdminNegativeTest-537197341-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "2f95123fa26d4061b5748fdb689e5985", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "980cb890-345b-4bf8-990a-a2faec78e49c", "external-id": "nsx-vlan-transportzone-965", "segmentation_id": 965, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc3739fab-ec", "ovs_interfaceid": "c3739fab-ec53-42cb-82cc-dd4c699fd80b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 910.325739] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:b9:8c:30', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '980cb890-345b-4bf8-990a-a2faec78e49c', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'c3739fab-ec53-42cb-82cc-dd4c699fd80b', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 910.333130] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Creating folder: Project (2f95123fa26d4061b5748fdb689e5985). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 910.333397] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-2d4575b8-9dcc-42a3-a6fd-80d6fe85ef87 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.344872] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Created folder: Project (2f95123fa26d4061b5748fdb689e5985) in parent group-v291016. [ 910.345081] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Creating folder: Instances. Parent ref: group-v291045. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 910.345302] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b6a156b1-5c57-4b16-9bc7-51fae1ae27df {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.356586] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Created folder: Instances in parent group-v291045. [ 910.356835] env[63593]: DEBUG oslo.service.loopingcall [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 910.357057] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 910.357263] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-743da2f5-be43-411f-b68b-cf3d8b6ad2fe {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.375200] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 910.375200] env[63593]: value = "task-1367980" [ 910.375200] env[63593]: _type = "Task" [ 910.375200] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 910.382432] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367980, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 910.418594] env[63593]: DEBUG nova.scheduler.client.report [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 910.576235] env[63593]: DEBUG nova.compute.manager [req-8e48437b-1816-4282-a469-c27b9c8b103f req-1c5840cc-c890-4f5b-a92c-6a5ad493dc36 service nova] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Received event network-changed-c3739fab-ec53-42cb-82cc-dd4c699fd80b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 910.576434] env[63593]: DEBUG nova.compute.manager [req-8e48437b-1816-4282-a469-c27b9c8b103f req-1c5840cc-c890-4f5b-a92c-6a5ad493dc36 service nova] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Refreshing instance network info cache due to event network-changed-c3739fab-ec53-42cb-82cc-dd4c699fd80b. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 910.576723] env[63593]: DEBUG oslo_concurrency.lockutils [req-8e48437b-1816-4282-a469-c27b9c8b103f req-1c5840cc-c890-4f5b-a92c-6a5ad493dc36 service nova] Acquiring lock "refresh_cache-fbd620a6-3743-4737-98b7-b13928b6587d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 910.576986] env[63593]: DEBUG oslo_concurrency.lockutils [req-8e48437b-1816-4282-a469-c27b9c8b103f req-1c5840cc-c890-4f5b-a92c-6a5ad493dc36 service nova] Acquired lock "refresh_cache-fbd620a6-3743-4737-98b7-b13928b6587d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 910.577322] env[63593]: DEBUG nova.network.neutron [req-8e48437b-1816-4282-a469-c27b9c8b103f req-1c5840cc-c890-4f5b-a92c-6a5ad493dc36 service nova] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Refreshing network info cache for port c3739fab-ec53-42cb-82cc-dd4c699fd80b {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 910.584219] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 910.611379] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 910.611612] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 910.611775] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 910.611958] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 910.612116] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 910.612262] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 910.612463] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 910.612612] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 910.612773] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 910.612929] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 910.613112] env[63593]: DEBUG nova.virt.hardware [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 910.614187] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9295054-ae4c-4dad-92df-d5662945a7fd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.622238] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6388ca3-8f9b-4066-a18b-96a47a0e855f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.884652] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367980, 'name': CreateVM_Task} progress is 99%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 910.924829] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.371s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 910.925440] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 910.928382] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 10.098s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 911.159302] env[63593]: DEBUG nova.network.neutron [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Successfully updated port: a1f7f8bf-af7d-4296-b234-000d10f51b2b {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 911.291195] env[63593]: DEBUG nova.network.neutron [req-8e48437b-1816-4282-a469-c27b9c8b103f req-1c5840cc-c890-4f5b-a92c-6a5ad493dc36 service nova] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Updated VIF entry in instance network info cache for port c3739fab-ec53-42cb-82cc-dd4c699fd80b. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 911.291578] env[63593]: DEBUG nova.network.neutron [req-8e48437b-1816-4282-a469-c27b9c8b103f req-1c5840cc-c890-4f5b-a92c-6a5ad493dc36 service nova] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Updating instance_info_cache with network_info: [{"id": "c3739fab-ec53-42cb-82cc-dd4c699fd80b", "address": "fa:16:3e:b9:8c:30", "network": {"id": "77bd00f0-884e-4553-b133-ebe7f63c5d20", "bridge": "br-int", "label": "tempest-VolumesAdminNegativeTest-537197341-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "2f95123fa26d4061b5748fdb689e5985", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "980cb890-345b-4bf8-990a-a2faec78e49c", "external-id": "nsx-vlan-transportzone-965", "segmentation_id": 965, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc3739fab-ec", "ovs_interfaceid": "c3739fab-ec53-42cb-82cc-dd4c699fd80b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 911.386148] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367980, 'name': CreateVM_Task, 'duration_secs': 0.558469} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 911.386688] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 911.387074] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 911.387286] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 911.387608] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 911.387886] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7a9ac6ea-8340-47f7-868f-d502d87337df {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.392612] env[63593]: DEBUG oslo_vmware.api [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Waiting for the task: (returnval){ [ 911.392612] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52862ba6-70c4-e029-e712-d910c4de72f9" [ 911.392612] env[63593]: _type = "Task" [ 911.392612] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 911.400776] env[63593]: DEBUG oslo_vmware.api [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52862ba6-70c4-e029-e712-d910c4de72f9, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 911.436293] env[63593]: DEBUG nova.compute.utils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 911.438657] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 911.438657] env[63593]: DEBUG nova.network.neutron [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 911.511789] env[63593]: DEBUG nova.policy [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '79470a9d8a6d4e76b7ca84948c4ea7df', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9f0780232f2a46db849695b5490bb673', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 911.663792] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "refresh_cache-268653cf-fcf9-4b50-9234-0b38fb9a2d95" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 911.663939] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired lock "refresh_cache-268653cf-fcf9-4b50-9234-0b38fb9a2d95" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 911.664373] env[63593]: DEBUG nova.network.neutron [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 911.725661] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13057b4e-7886-4a49-89ee-5953bb81fb08 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.734667] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbc29fe2-13ca-4c97-b404-0154c6c4090d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.766928] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4541e711-b9c5-482e-8634-f68d1684da70 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.775764] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24ad58c3-9bcf-412e-bb16-7f893b84dcc3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 911.789265] env[63593]: DEBUG nova.compute.provider_tree [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 911.794012] env[63593]: DEBUG oslo_concurrency.lockutils [req-8e48437b-1816-4282-a469-c27b9c8b103f req-1c5840cc-c890-4f5b-a92c-6a5ad493dc36 service nova] Releasing lock "refresh_cache-fbd620a6-3743-4737-98b7-b13928b6587d" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 911.824051] env[63593]: DEBUG nova.network.neutron [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Successfully created port: d4878310-3636-463b-bf00-ae1e5cdeadf6 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 911.903451] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 911.903712] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 911.903925] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 911.941189] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 912.089271] env[63593]: DEBUG nova.network.neutron [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Successfully created port: ca1adb70-3370-4151-9cc9-3db06afffc86 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 912.198922] env[63593]: DEBUG nova.network.neutron [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 912.295399] env[63593]: DEBUG nova.scheduler.client.report [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 912.390264] env[63593]: DEBUG nova.network.neutron [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Updating instance_info_cache with network_info: [{"id": "a1f7f8bf-af7d-4296-b234-000d10f51b2b", "address": "fa:16:3e:6a:d2:de", "network": {"id": "c440ce66-80a9-4360-813a-816782de8ecd", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-114095660-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "60d47be26aac44d198ca43a997485f7c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f77ff7a1-209c-4f3f-b2a0-fd817741e739", "external-id": "nsx-vlan-transportzone-935", "segmentation_id": 935, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapa1f7f8bf-af", "ovs_interfaceid": "a1f7f8bf-af7d-4296-b234-000d10f51b2b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 912.657876] env[63593]: DEBUG nova.compute.manager [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Received event network-vif-plugged-a1f7f8bf-af7d-4296-b234-000d10f51b2b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 912.658104] env[63593]: DEBUG oslo_concurrency.lockutils [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] Acquiring lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 912.658327] env[63593]: DEBUG oslo_concurrency.lockutils [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 912.658501] env[63593]: DEBUG oslo_concurrency.lockutils [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 912.658666] env[63593]: DEBUG nova.compute.manager [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] No waiting events found dispatching network-vif-plugged-a1f7f8bf-af7d-4296-b234-000d10f51b2b {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 912.658925] env[63593]: WARNING nova.compute.manager [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Received unexpected event network-vif-plugged-a1f7f8bf-af7d-4296-b234-000d10f51b2b for instance with vm_state building and task_state spawning. [ 912.659031] env[63593]: DEBUG nova.compute.manager [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Received event network-changed-a1f7f8bf-af7d-4296-b234-000d10f51b2b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 912.659193] env[63593]: DEBUG nova.compute.manager [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Refreshing instance network info cache due to event network-changed-a1f7f8bf-af7d-4296-b234-000d10f51b2b. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 912.659358] env[63593]: DEBUG oslo_concurrency.lockutils [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] Acquiring lock "refresh_cache-268653cf-fcf9-4b50-9234-0b38fb9a2d95" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 912.800969] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.872s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 912.802063] env[63593]: ERROR nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 912.802063] env[63593]: Faults: ['InvalidArgument'] [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Traceback (most recent call last): [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] self.driver.spawn(context, instance, image_meta, [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] self._vmops.spawn(context, instance, image_meta, injected_files, [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] self._fetch_image_if_missing(context, vi) [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] image_cache(vi, tmp_image_ds_loc) [ 912.802063] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] vm_util.copy_virtual_disk( [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] session._wait_for_task(vmdk_copy_task) [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] return self.wait_for_task(task_ref) [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] return evt.wait() [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] result = hub.switch() [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] return self.greenlet.switch() [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 912.802418] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] self.f(*self.args, **self.kw) [ 912.802719] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 912.802719] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] raise exceptions.translate_fault(task_info.error) [ 912.802719] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 912.802719] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Faults: ['InvalidArgument'] [ 912.802719] env[63593]: ERROR nova.compute.manager [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] [ 912.802719] env[63593]: DEBUG nova.compute.utils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 912.803609] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.244s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 912.806193] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Build of instance 94ff0182-d1ce-405f-bdb5-a2399bd0f4de was re-scheduled: A specified parameter was not correct: fileType [ 912.806193] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 912.806569] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 912.806814] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "refresh_cache-94ff0182-d1ce-405f-bdb5-a2399bd0f4de" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 912.806967] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired lock "refresh_cache-94ff0182-d1ce-405f-bdb5-a2399bd0f4de" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 912.807312] env[63593]: DEBUG nova.network.neutron [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 912.893232] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Releasing lock "refresh_cache-268653cf-fcf9-4b50-9234-0b38fb9a2d95" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 912.893619] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Instance network_info: |[{"id": "a1f7f8bf-af7d-4296-b234-000d10f51b2b", "address": "fa:16:3e:6a:d2:de", "network": {"id": "c440ce66-80a9-4360-813a-816782de8ecd", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-114095660-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "60d47be26aac44d198ca43a997485f7c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f77ff7a1-209c-4f3f-b2a0-fd817741e739", "external-id": "nsx-vlan-transportzone-935", "segmentation_id": 935, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapa1f7f8bf-af", "ovs_interfaceid": "a1f7f8bf-af7d-4296-b234-000d10f51b2b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 912.893969] env[63593]: DEBUG oslo_concurrency.lockutils [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] Acquired lock "refresh_cache-268653cf-fcf9-4b50-9234-0b38fb9a2d95" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 912.894116] env[63593]: DEBUG nova.network.neutron [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Refreshing network info cache for port a1f7f8bf-af7d-4296-b234-000d10f51b2b {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 912.895356] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:6a:d2:de', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f77ff7a1-209c-4f3f-b2a0-fd817741e739', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'a1f7f8bf-af7d-4296-b234-000d10f51b2b', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 912.902799] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating folder: Project (60d47be26aac44d198ca43a997485f7c). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 912.906020] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b90e8e5c-e42f-4886-8a44-5b22642ae2a6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.917828] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Created folder: Project (60d47be26aac44d198ca43a997485f7c) in parent group-v291016. [ 912.918026] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating folder: Instances. Parent ref: group-v291048. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 912.918501] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-262ba944-e0a5-43d5-8a04-7e715a88bec5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.926602] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Created folder: Instances in parent group-v291048. [ 912.926835] env[63593]: DEBUG oslo.service.loopingcall [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 912.927026] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 912.927229] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-24173dd5-3e36-4f64-b08f-bc283353b9d2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.947578] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 912.947578] env[63593]: value = "task-1367983" [ 912.947578] env[63593]: _type = "Task" [ 912.947578] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 912.951684] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 912.959035] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367983, 'name': CreateVM_Task} progress is 5%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 912.979089] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 912.979346] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 912.979499] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 912.979676] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 912.979818] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 912.979993] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 912.980353] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 912.980521] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 912.980687] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 912.980847] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 912.981041] env[63593]: DEBUG nova.virt.hardware [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 912.982110] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ea6816b-190c-43c7-aa15-7a7e564e317a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.991958] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc71743b-14ea-41c8-a1cc-e3d9cf8da21f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.146777] env[63593]: DEBUG nova.network.neutron [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Updated VIF entry in instance network info cache for port a1f7f8bf-af7d-4296-b234-000d10f51b2b. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 913.147681] env[63593]: DEBUG nova.network.neutron [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Updating instance_info_cache with network_info: [{"id": "a1f7f8bf-af7d-4296-b234-000d10f51b2b", "address": "fa:16:3e:6a:d2:de", "network": {"id": "c440ce66-80a9-4360-813a-816782de8ecd", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-114095660-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "60d47be26aac44d198ca43a997485f7c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f77ff7a1-209c-4f3f-b2a0-fd817741e739", "external-id": "nsx-vlan-transportzone-935", "segmentation_id": 935, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapa1f7f8bf-af", "ovs_interfaceid": "a1f7f8bf-af7d-4296-b234-000d10f51b2b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 913.330782] env[63593]: DEBUG nova.network.neutron [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 913.379876] env[63593]: DEBUG nova.network.neutron [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 913.459306] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367983, 'name': CreateVM_Task, 'duration_secs': 0.311003} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 913.459578] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 913.460341] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 913.460549] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 913.460893] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 913.461190] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a365f861-70a0-4af1-82a6-8e59e9e9e509 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.465813] env[63593]: DEBUG oslo_vmware.api [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 913.465813] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5276fa4c-18ae-a410-1988-4387f5498b02" [ 913.465813] env[63593]: _type = "Task" [ 913.465813] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 913.476564] env[63593]: DEBUG oslo_vmware.api [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5276fa4c-18ae-a410-1988-4387f5498b02, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 913.538300] env[63593]: DEBUG nova.compute.manager [req-70e79980-9b73-4ded-a409-1fb7a97f205a req-05554078-79be-41cb-9285-15758bd8bae2 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Received event network-vif-plugged-d4878310-3636-463b-bf00-ae1e5cdeadf6 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 913.538910] env[63593]: DEBUG oslo_concurrency.lockutils [req-70e79980-9b73-4ded-a409-1fb7a97f205a req-05554078-79be-41cb-9285-15758bd8bae2 service nova] Acquiring lock "328b91c6-4ae4-442f-ac87-519658a812ff-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 913.539024] env[63593]: DEBUG oslo_concurrency.lockutils [req-70e79980-9b73-4ded-a409-1fb7a97f205a req-05554078-79be-41cb-9285-15758bd8bae2 service nova] Lock "328b91c6-4ae4-442f-ac87-519658a812ff-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 913.540112] env[63593]: DEBUG oslo_concurrency.lockutils [req-70e79980-9b73-4ded-a409-1fb7a97f205a req-05554078-79be-41cb-9285-15758bd8bae2 service nova] Lock "328b91c6-4ae4-442f-ac87-519658a812ff-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 913.540112] env[63593]: DEBUG nova.compute.manager [req-70e79980-9b73-4ded-a409-1fb7a97f205a req-05554078-79be-41cb-9285-15758bd8bae2 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] No waiting events found dispatching network-vif-plugged-d4878310-3636-463b-bf00-ae1e5cdeadf6 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 913.540112] env[63593]: WARNING nova.compute.manager [req-70e79980-9b73-4ded-a409-1fb7a97f205a req-05554078-79be-41cb-9285-15758bd8bae2 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Received unexpected event network-vif-plugged-d4878310-3636-463b-bf00-ae1e5cdeadf6 for instance with vm_state building and task_state spawning. [ 913.578667] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3dd9affe-72b0-47b6-859b-82006aa79aac {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.587394] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29724d85-86a0-4fda-82c9-a27db9943679 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.616900] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68064ae5-7aec-40f1-aac7-b307d5a25376 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.623994] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-415675a2-b739-46f9-a9ac-7f07ce20e0de {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 913.628308] env[63593]: DEBUG nova.network.neutron [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Successfully updated port: d4878310-3636-463b-bf00-ae1e5cdeadf6 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 913.640827] env[63593]: DEBUG nova.compute.provider_tree [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 913.651322] env[63593]: DEBUG oslo_concurrency.lockutils [req-100a9d30-85d6-4a79-8c75-8e2f3bba23ce req-31199510-66ca-413c-859f-4abca28d8b9f service nova] Releasing lock "refresh_cache-268653cf-fcf9-4b50-9234-0b38fb9a2d95" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 913.881955] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Releasing lock "refresh_cache-94ff0182-d1ce-405f-bdb5-a2399bd0f4de" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 913.882234] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 913.882430] env[63593]: DEBUG nova.compute.manager [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 94ff0182-d1ce-405f-bdb5-a2399bd0f4de] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 913.976277] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 913.976555] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 913.976795] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 914.145935] env[63593]: DEBUG nova.scheduler.client.report [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 914.650684] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.847s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 914.651390] env[63593]: ERROR nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11, please check neutron logs for more information. [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Traceback (most recent call last): [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self.driver.spawn(context, instance, image_meta, [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self._vmops.spawn(context, instance, image_meta, injected_files, [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] vm_ref = self.build_virtual_machine(instance, [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] vif_infos = vmwarevif.get_vif_info(self._session, [ 914.651390] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] for vif in network_info: [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] return self._sync_wrapper(fn, *args, **kwargs) [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self.wait() [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self[:] = self._gt.wait() [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] return self._exit_event.wait() [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] current.throw(*self._exc) [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 914.651789] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] result = function(*args, **kwargs) [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] return func(*args, **kwargs) [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] raise e [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] nwinfo = self.network_api.allocate_for_instance( [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] created_port_ids = self._update_ports_for_instance( [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] with excutils.save_and_reraise_exception(): [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] self.force_reraise() [ 914.652136] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 914.652482] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] raise self.value [ 914.652482] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 914.652482] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] updated_port = self._update_port( [ 914.652482] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 914.652482] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] _ensure_no_port_binding_failure(port) [ 914.652482] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 914.652482] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] raise exception.PortBindingFailed(port_id=port['id']) [ 914.652482] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] nova.exception.PortBindingFailed: Binding failed for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11, please check neutron logs for more information. [ 914.652482] env[63593]: ERROR nova.compute.manager [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] [ 914.652482] env[63593]: DEBUG nova.compute.utils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Binding failed for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11, please check neutron logs for more information. {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 914.656169] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Build of instance ef8646c6-7ead-4c9f-8e8f-85c758a35006 was re-scheduled: Binding failed for port fdb0f1c9-485b-4b15-94d3-e7d274b52f11, please check neutron logs for more information. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 914.656580] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 914.656823] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Acquiring lock "refresh_cache-ef8646c6-7ead-4c9f-8e8f-85c758a35006" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 914.656989] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Acquired lock "refresh_cache-ef8646c6-7ead-4c9f-8e8f-85c758a35006" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 914.657165] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 914.916360] env[63593]: INFO nova.scheduler.client.report [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Deleted allocations for instance 94ff0182-d1ce-405f-bdb5-a2399bd0f4de [ 915.174930] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 915.257756] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 915.426939] env[63593]: DEBUG oslo_concurrency.lockutils [None req-81077f2a-ac0f-4840-b7d4-139549a11230 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "94ff0182-d1ce-405f-bdb5-a2399bd0f4de" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 193.992s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 915.659790] env[63593]: DEBUG nova.compute.manager [req-d6f19dae-5681-4406-ad11-79ac96a34379 req-cbccdffa-280d-4440-bc62-ea87ce30cbec service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Received event network-changed-d4878310-3636-463b-bf00-ae1e5cdeadf6 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 915.660076] env[63593]: DEBUG nova.compute.manager [req-d6f19dae-5681-4406-ad11-79ac96a34379 req-cbccdffa-280d-4440-bc62-ea87ce30cbec service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Refreshing instance network info cache due to event network-changed-d4878310-3636-463b-bf00-ae1e5cdeadf6. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 915.660303] env[63593]: DEBUG oslo_concurrency.lockutils [req-d6f19dae-5681-4406-ad11-79ac96a34379 req-cbccdffa-280d-4440-bc62-ea87ce30cbec service nova] Acquiring lock "refresh_cache-328b91c6-4ae4-442f-ac87-519658a812ff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 915.660442] env[63593]: DEBUG oslo_concurrency.lockutils [req-d6f19dae-5681-4406-ad11-79ac96a34379 req-cbccdffa-280d-4440-bc62-ea87ce30cbec service nova] Acquired lock "refresh_cache-328b91c6-4ae4-442f-ac87-519658a812ff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 915.660597] env[63593]: DEBUG nova.network.neutron [req-d6f19dae-5681-4406-ad11-79ac96a34379 req-cbccdffa-280d-4440-bc62-ea87ce30cbec service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Refreshing network info cache for port d4878310-3636-463b-bf00-ae1e5cdeadf6 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 915.760195] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Releasing lock "refresh_cache-ef8646c6-7ead-4c9f-8e8f-85c758a35006" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 915.760436] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 915.760612] env[63593]: DEBUG nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 915.760774] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 915.776265] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 915.901556] env[63593]: DEBUG nova.network.neutron [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Successfully updated port: ca1adb70-3370-4151-9cc9-3db06afffc86 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 915.930129] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 916.196108] env[63593]: DEBUG nova.network.neutron [req-d6f19dae-5681-4406-ad11-79ac96a34379 req-cbccdffa-280d-4440-bc62-ea87ce30cbec service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 916.279265] env[63593]: DEBUG nova.network.neutron [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 916.313090] env[63593]: DEBUG nova.network.neutron [req-d6f19dae-5681-4406-ad11-79ac96a34379 req-cbccdffa-280d-4440-bc62-ea87ce30cbec service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 916.405093] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "refresh_cache-328b91c6-4ae4-442f-ac87-519658a812ff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 916.452483] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 916.452749] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 916.454157] env[63593]: INFO nova.compute.claims [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 916.782074] env[63593]: INFO nova.compute.manager [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] [instance: ef8646c6-7ead-4c9f-8e8f-85c758a35006] Took 1.02 seconds to deallocate network for instance. [ 916.815947] env[63593]: DEBUG oslo_concurrency.lockutils [req-d6f19dae-5681-4406-ad11-79ac96a34379 req-cbccdffa-280d-4440-bc62-ea87ce30cbec service nova] Releasing lock "refresh_cache-328b91c6-4ae4-442f-ac87-519658a812ff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 916.816620] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquired lock "refresh_cache-328b91c6-4ae4-442f-ac87-519658a812ff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 916.816841] env[63593]: DEBUG nova.network.neutron [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 917.366967] env[63593]: DEBUG nova.network.neutron [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 917.655210] env[63593]: DEBUG nova.network.neutron [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Updating instance_info_cache with network_info: [{"id": "d4878310-3636-463b-bf00-ae1e5cdeadf6", "address": "fa:16:3e:57:2b:b2", "network": {"id": "cdc0403c-343b-4c78-ba70-ba0be8768a01", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-888725264", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.103", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "9f0780232f2a46db849695b5490bb673", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a0d2101e-2d93-4310-a242-af2d9ecdaf9b", "external-id": "nsx-vlan-transportzone-121", "segmentation_id": 121, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd4878310-36", "ovs_interfaceid": "d4878310-3636-463b-bf00-ae1e5cdeadf6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "ca1adb70-3370-4151-9cc9-3db06afffc86", "address": "fa:16:3e:35:96:48", "network": {"id": "2a1a483c-9f66-4142-88f8-389ee17f6ba8", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-1207242292", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.139", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "9f0780232f2a46db849695b5490bb673", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "1ce8361b-fd8e-4971-a37f-b84a4f77db19", "external-id": "nsx-vlan-transportzone-255", "segmentation_id": 255, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapca1adb70-33", "ovs_interfaceid": "ca1adb70-3370-4151-9cc9-3db06afffc86", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 917.686227] env[63593]: DEBUG nova.compute.manager [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Received event network-vif-plugged-ca1adb70-3370-4151-9cc9-3db06afffc86 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 917.686454] env[63593]: DEBUG oslo_concurrency.lockutils [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] Acquiring lock "328b91c6-4ae4-442f-ac87-519658a812ff-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 917.686658] env[63593]: DEBUG oslo_concurrency.lockutils [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] Lock "328b91c6-4ae4-442f-ac87-519658a812ff-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 917.686832] env[63593]: DEBUG oslo_concurrency.lockutils [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] Lock "328b91c6-4ae4-442f-ac87-519658a812ff-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 917.686998] env[63593]: DEBUG nova.compute.manager [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] No waiting events found dispatching network-vif-plugged-ca1adb70-3370-4151-9cc9-3db06afffc86 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 917.687181] env[63593]: WARNING nova.compute.manager [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Received unexpected event network-vif-plugged-ca1adb70-3370-4151-9cc9-3db06afffc86 for instance with vm_state building and task_state spawning. [ 917.687340] env[63593]: DEBUG nova.compute.manager [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Received event network-changed-ca1adb70-3370-4151-9cc9-3db06afffc86 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 917.687489] env[63593]: DEBUG nova.compute.manager [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Refreshing instance network info cache due to event network-changed-ca1adb70-3370-4151-9cc9-3db06afffc86. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 917.687684] env[63593]: DEBUG oslo_concurrency.lockutils [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] Acquiring lock "refresh_cache-328b91c6-4ae4-442f-ac87-519658a812ff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 917.698853] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-949905ee-12fc-4445-bb6d-a1e1b3199e1c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.706288] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ccbb20f-320d-4290-994b-01c7687eb286 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.735097] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcee1b05-9344-417d-9851-1997d6802df1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.741958] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b3e7703-8122-4d02-84ed-5a03cd94b8fa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.754546] env[63593]: DEBUG nova.compute.provider_tree [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 917.807920] env[63593]: INFO nova.scheduler.client.report [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Deleted allocations for instance ef8646c6-7ead-4c9f-8e8f-85c758a35006 [ 917.991498] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "4e312fd5-01bf-4bd4-8a61-26624ae05036" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 918.158220] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Releasing lock "refresh_cache-328b91c6-4ae4-442f-ac87-519658a812ff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 918.158618] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Instance network_info: |[{"id": "d4878310-3636-463b-bf00-ae1e5cdeadf6", "address": "fa:16:3e:57:2b:b2", "network": {"id": "cdc0403c-343b-4c78-ba70-ba0be8768a01", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-888725264", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.103", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "9f0780232f2a46db849695b5490bb673", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a0d2101e-2d93-4310-a242-af2d9ecdaf9b", "external-id": "nsx-vlan-transportzone-121", "segmentation_id": 121, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd4878310-36", "ovs_interfaceid": "d4878310-3636-463b-bf00-ae1e5cdeadf6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "ca1adb70-3370-4151-9cc9-3db06afffc86", "address": "fa:16:3e:35:96:48", "network": {"id": "2a1a483c-9f66-4142-88f8-389ee17f6ba8", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-1207242292", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.139", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "9f0780232f2a46db849695b5490bb673", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "1ce8361b-fd8e-4971-a37f-b84a4f77db19", "external-id": "nsx-vlan-transportzone-255", "segmentation_id": 255, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapca1adb70-33", "ovs_interfaceid": "ca1adb70-3370-4151-9cc9-3db06afffc86", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 918.158953] env[63593]: DEBUG oslo_concurrency.lockutils [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] Acquired lock "refresh_cache-328b91c6-4ae4-442f-ac87-519658a812ff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 918.159152] env[63593]: DEBUG nova.network.neutron [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Refreshing network info cache for port ca1adb70-3370-4151-9cc9-3db06afffc86 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 918.160731] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:57:2b:b2', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a0d2101e-2d93-4310-a242-af2d9ecdaf9b', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'd4878310-3636-463b-bf00-ae1e5cdeadf6', 'vif_model': 'vmxnet3'}, {'network_name': 'br-int', 'mac_address': 'fa:16:3e:35:96:48', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '1ce8361b-fd8e-4971-a37f-b84a4f77db19', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'ca1adb70-3370-4151-9cc9-3db06afffc86', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 918.169400] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Creating folder: Project (9f0780232f2a46db849695b5490bb673). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 918.172323] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-e55268be-3e06-4219-83ab-76fe8811c364 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.183493] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Created folder: Project (9f0780232f2a46db849695b5490bb673) in parent group-v291016. [ 918.183691] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Creating folder: Instances. Parent ref: group-v291051. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 918.183918] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-a374d356-0604-470f-a063-1da985449d04 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.193378] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Created folder: Instances in parent group-v291051. [ 918.193654] env[63593]: DEBUG oslo.service.loopingcall [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 918.193872] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 918.194117] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-5de919a2-901b-401f-8b97-5f7630c85875 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.215026] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 918.215026] env[63593]: value = "task-1367986" [ 918.215026] env[63593]: _type = "Task" [ 918.215026] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 918.223048] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367986, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 918.256897] env[63593]: DEBUG nova.scheduler.client.report [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 918.315630] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a69c974-048a-4f8c-b1b7-6d42b20ed8bc tempest-ServerTagsTestJSON-891254822 tempest-ServerTagsTestJSON-891254822-project-member] Lock "ef8646c6-7ead-4c9f-8e8f-85c758a35006" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 132.822s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 918.469787] env[63593]: DEBUG nova.network.neutron [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Updated VIF entry in instance network info cache for port ca1adb70-3370-4151-9cc9-3db06afffc86. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 918.470376] env[63593]: DEBUG nova.network.neutron [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Updating instance_info_cache with network_info: [{"id": "d4878310-3636-463b-bf00-ae1e5cdeadf6", "address": "fa:16:3e:57:2b:b2", "network": {"id": "cdc0403c-343b-4c78-ba70-ba0be8768a01", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-888725264", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.103", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "9f0780232f2a46db849695b5490bb673", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a0d2101e-2d93-4310-a242-af2d9ecdaf9b", "external-id": "nsx-vlan-transportzone-121", "segmentation_id": 121, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd4878310-36", "ovs_interfaceid": "d4878310-3636-463b-bf00-ae1e5cdeadf6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "ca1adb70-3370-4151-9cc9-3db06afffc86", "address": "fa:16:3e:35:96:48", "network": {"id": "2a1a483c-9f66-4142-88f8-389ee17f6ba8", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-1207242292", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.139", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "9f0780232f2a46db849695b5490bb673", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "1ce8361b-fd8e-4971-a37f-b84a4f77db19", "external-id": "nsx-vlan-transportzone-255", "segmentation_id": 255, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapca1adb70-33", "ovs_interfaceid": "ca1adb70-3370-4151-9cc9-3db06afffc86", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 918.725358] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367986, 'name': CreateVM_Task, 'duration_secs': 0.322844} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 918.725648] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 918.726519] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 918.726804] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 918.727229] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 918.727561] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5a189a75-ec06-4a67-bc95-f97f552a5aec {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.731873] env[63593]: DEBUG oslo_vmware.api [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Waiting for the task: (returnval){ [ 918.731873] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]528c5e3e-2a49-eb6e-24ce-7a838a811e1f" [ 918.731873] env[63593]: _type = "Task" [ 918.731873] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 918.739952] env[63593]: DEBUG oslo_vmware.api [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]528c5e3e-2a49-eb6e-24ce-7a838a811e1f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 918.761093] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.308s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 918.761625] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 918.817797] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 918.972939] env[63593]: DEBUG oslo_concurrency.lockutils [req-fbdb1081-e699-4300-8680-a16ad90b5bf0 req-d5149cf6-3b59-4384-8ed7-512ada437206 service nova] Releasing lock "refresh_cache-328b91c6-4ae4-442f-ac87-519658a812ff" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 919.243391] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 919.243795] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 919.244070] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 919.267824] env[63593]: DEBUG nova.compute.utils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 919.272586] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 919.275023] env[63593]: DEBUG nova.network.neutron [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 919.345841] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 919.346302] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 919.347835] env[63593]: INFO nova.compute.claims [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 919.434158] env[63593]: DEBUG nova.policy [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd5f23913d4a1411282f081fb5ed46a11', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '220c072768c1441fbfe88f849cd6eb03', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 919.779783] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 919.791156] env[63593]: DEBUG nova.network.neutron [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Successfully created port: 1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 920.624229] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abd1eba4-3e8a-4cda-b74c-1769bc475169 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.632117] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33123f3a-df1f-46ff-b74a-b34ab4496763 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.662890] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db8cd039-ae66-45ac-8de0-699f1738a9be {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.670521] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5cb8827b-674c-44eb-aef1-01349bac2170 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.683766] env[63593]: DEBUG nova.compute.provider_tree [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 920.792019] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 920.817089] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 920.817089] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 920.817089] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 920.817089] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 920.817089] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 920.817429] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 920.817890] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 920.818176] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 920.818455] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 920.818719] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 920.818995] env[63593]: DEBUG nova.virt.hardware [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 920.819988] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ed276fa-32bf-4065-8b35-bceb3552dc1a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.828521] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91eadc38-c931-40c7-a5e5-2a2a245dd76e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.186694] env[63593]: DEBUG nova.scheduler.client.report [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 921.309207] env[63593]: DEBUG nova.compute.manager [req-7d54c6c4-1372-441f-b4da-aad660e60dff req-3c66fd29-e199-4e92-819b-ce6feb440dff service nova] [instance: b053676a-81b8-4345-b257-1d67976fd077] Received event network-vif-plugged-1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 921.309474] env[63593]: DEBUG oslo_concurrency.lockutils [req-7d54c6c4-1372-441f-b4da-aad660e60dff req-3c66fd29-e199-4e92-819b-ce6feb440dff service nova] Acquiring lock "b053676a-81b8-4345-b257-1d67976fd077-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 921.310137] env[63593]: DEBUG oslo_concurrency.lockutils [req-7d54c6c4-1372-441f-b4da-aad660e60dff req-3c66fd29-e199-4e92-819b-ce6feb440dff service nova] Lock "b053676a-81b8-4345-b257-1d67976fd077-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 921.310315] env[63593]: DEBUG oslo_concurrency.lockutils [req-7d54c6c4-1372-441f-b4da-aad660e60dff req-3c66fd29-e199-4e92-819b-ce6feb440dff service nova] Lock "b053676a-81b8-4345-b257-1d67976fd077-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 921.310484] env[63593]: DEBUG nova.compute.manager [req-7d54c6c4-1372-441f-b4da-aad660e60dff req-3c66fd29-e199-4e92-819b-ce6feb440dff service nova] [instance: b053676a-81b8-4345-b257-1d67976fd077] No waiting events found dispatching network-vif-plugged-1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 921.310646] env[63593]: WARNING nova.compute.manager [req-7d54c6c4-1372-441f-b4da-aad660e60dff req-3c66fd29-e199-4e92-819b-ce6feb440dff service nova] [instance: b053676a-81b8-4345-b257-1d67976fd077] Received unexpected event network-vif-plugged-1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee for instance with vm_state building and task_state spawning. [ 921.691154] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.345s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 921.691705] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 921.784824] env[63593]: DEBUG nova.network.neutron [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Successfully updated port: 1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 921.831943] env[63593]: DEBUG nova.compute.manager [req-e78f5158-9663-4a09-a4d3-015a4333db67 req-44cfed8c-2295-4f64-b7e1-1af262a95d11 service nova] [instance: b053676a-81b8-4345-b257-1d67976fd077] Received event network-changed-1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 921.832104] env[63593]: DEBUG nova.compute.manager [req-e78f5158-9663-4a09-a4d3-015a4333db67 req-44cfed8c-2295-4f64-b7e1-1af262a95d11 service nova] [instance: b053676a-81b8-4345-b257-1d67976fd077] Refreshing instance network info cache due to event network-changed-1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 921.832324] env[63593]: DEBUG oslo_concurrency.lockutils [req-e78f5158-9663-4a09-a4d3-015a4333db67 req-44cfed8c-2295-4f64-b7e1-1af262a95d11 service nova] Acquiring lock "refresh_cache-b053676a-81b8-4345-b257-1d67976fd077" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 921.832465] env[63593]: DEBUG oslo_concurrency.lockutils [req-e78f5158-9663-4a09-a4d3-015a4333db67 req-44cfed8c-2295-4f64-b7e1-1af262a95d11 service nova] Acquired lock "refresh_cache-b053676a-81b8-4345-b257-1d67976fd077" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 921.832621] env[63593]: DEBUG nova.network.neutron [req-e78f5158-9663-4a09-a4d3-015a4333db67 req-44cfed8c-2295-4f64-b7e1-1af262a95d11 service nova] [instance: b053676a-81b8-4345-b257-1d67976fd077] Refreshing network info cache for port 1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 922.197277] env[63593]: DEBUG nova.compute.utils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 922.198552] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 922.198722] env[63593]: DEBUG nova.network.neutron [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 922.235801] env[63593]: DEBUG nova.policy [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a42fbf6f3650495eb40dff27e91a6a5f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7a529e687cb043acae6837e7f4f25a25', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 922.288340] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "refresh_cache-b053676a-81b8-4345-b257-1d67976fd077" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 922.364437] env[63593]: DEBUG nova.network.neutron [req-e78f5158-9663-4a09-a4d3-015a4333db67 req-44cfed8c-2295-4f64-b7e1-1af262a95d11 service nova] [instance: b053676a-81b8-4345-b257-1d67976fd077] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 922.435760] env[63593]: DEBUG nova.network.neutron [req-e78f5158-9663-4a09-a4d3-015a4333db67 req-44cfed8c-2295-4f64-b7e1-1af262a95d11 service nova] [instance: b053676a-81b8-4345-b257-1d67976fd077] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 922.504859] env[63593]: DEBUG nova.network.neutron [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Successfully created port: bad74310-1723-4f07-b2ed-798288e20247 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 922.702023] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 922.941988] env[63593]: DEBUG oslo_concurrency.lockutils [req-e78f5158-9663-4a09-a4d3-015a4333db67 req-44cfed8c-2295-4f64-b7e1-1af262a95d11 service nova] Releasing lock "refresh_cache-b053676a-81b8-4345-b257-1d67976fd077" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 922.942406] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "refresh_cache-b053676a-81b8-4345-b257-1d67976fd077" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 922.942565] env[63593]: DEBUG nova.network.neutron [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 923.472355] env[63593]: DEBUG nova.network.neutron [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 923.602567] env[63593]: DEBUG nova.network.neutron [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Updating instance_info_cache with network_info: [{"id": "1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee", "address": "fa:16:3e:0f:cf:8c", "network": {"id": "fb445387-b1c5-45bc-a832-c443e70d8604", "bridge": "br-int", "label": "tempest-ServerDiskConfigTestJSON-429488907-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "220c072768c1441fbfe88f849cd6eb03", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bafe8721-91d4-4127-b215-d9e8e27947dc", "external-id": "nsx-vlan-transportzone-680", "segmentation_id": 680, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1c1b0a6d-7c", "ovs_interfaceid": "1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 923.712702] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 923.742258] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 923.742519] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 923.742674] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 923.742852] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 923.743020] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 923.743263] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 923.743478] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 923.743636] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 923.745112] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 923.745348] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 923.745534] env[63593]: DEBUG nova.virt.hardware [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 923.746438] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc0bb383-a6dc-46cd-a792-8d2c3c2c0c00 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.756283] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68bf0715-51a3-4399-8a3d-234c252f589e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.105646] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "refresh_cache-b053676a-81b8-4345-b257-1d67976fd077" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 924.105981] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Instance network_info: |[{"id": "1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee", "address": "fa:16:3e:0f:cf:8c", "network": {"id": "fb445387-b1c5-45bc-a832-c443e70d8604", "bridge": "br-int", "label": "tempest-ServerDiskConfigTestJSON-429488907-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "220c072768c1441fbfe88f849cd6eb03", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bafe8721-91d4-4127-b215-d9e8e27947dc", "external-id": "nsx-vlan-transportzone-680", "segmentation_id": 680, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1c1b0a6d-7c", "ovs_interfaceid": "1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 924.107284] env[63593]: DEBUG nova.compute.manager [req-9bb237d7-d572-4e94-99fb-10e15a420f27 req-a6a26b2a-b34f-4e30-9564-c27c584b7199 service nova] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Received event network-vif-plugged-bad74310-1723-4f07-b2ed-798288e20247 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 924.107394] env[63593]: DEBUG oslo_concurrency.lockutils [req-9bb237d7-d572-4e94-99fb-10e15a420f27 req-a6a26b2a-b34f-4e30-9564-c27c584b7199 service nova] Acquiring lock "cf724d13-e34e-4a17-9b54-56190891d5db-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 924.107597] env[63593]: DEBUG oslo_concurrency.lockutils [req-9bb237d7-d572-4e94-99fb-10e15a420f27 req-a6a26b2a-b34f-4e30-9564-c27c584b7199 service nova] Lock "cf724d13-e34e-4a17-9b54-56190891d5db-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 924.107757] env[63593]: DEBUG oslo_concurrency.lockutils [req-9bb237d7-d572-4e94-99fb-10e15a420f27 req-a6a26b2a-b34f-4e30-9564-c27c584b7199 service nova] Lock "cf724d13-e34e-4a17-9b54-56190891d5db-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 924.107919] env[63593]: DEBUG nova.compute.manager [req-9bb237d7-d572-4e94-99fb-10e15a420f27 req-a6a26b2a-b34f-4e30-9564-c27c584b7199 service nova] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] No waiting events found dispatching network-vif-plugged-bad74310-1723-4f07-b2ed-798288e20247 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 924.108095] env[63593]: WARNING nova.compute.manager [req-9bb237d7-d572-4e94-99fb-10e15a420f27 req-a6a26b2a-b34f-4e30-9564-c27c584b7199 service nova] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Received unexpected event network-vif-plugged-bad74310-1723-4f07-b2ed-798288e20247 for instance with vm_state building and task_state spawning. [ 924.108528] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:0f:cf:8c', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'bafe8721-91d4-4127-b215-d9e8e27947dc', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '1c1b0a6d-7c1d-4e61-9a8f-66cea40a1aee', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 924.118987] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Creating folder: Project (220c072768c1441fbfe88f849cd6eb03). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 924.119590] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-5d7bfdd7-d021-4833-9a55-175538937257 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.131570] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Created folder: Project (220c072768c1441fbfe88f849cd6eb03) in parent group-v291016. [ 924.131749] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Creating folder: Instances. Parent ref: group-v291054. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 924.131970] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-c9dbb360-c2a0-4fc7-bfc8-2688545f4659 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.139890] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Created folder: Instances in parent group-v291054. [ 924.140127] env[63593]: DEBUG oslo.service.loopingcall [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 924.140309] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b053676a-81b8-4345-b257-1d67976fd077] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 924.140499] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-560571ea-7cbd-46a0-9542-4e9392c4a7c9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.163330] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 924.163330] env[63593]: value = "task-1367989" [ 924.163330] env[63593]: _type = "Task" [ 924.163330] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 924.163330] env[63593]: DEBUG nova.network.neutron [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Successfully updated port: bad74310-1723-4f07-b2ed-798288e20247 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 924.174067] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367989, 'name': CreateVM_Task} progress is 6%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 924.666062] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquiring lock "refresh_cache-cf724d13-e34e-4a17-9b54-56190891d5db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 924.666062] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquired lock "refresh_cache-cf724d13-e34e-4a17-9b54-56190891d5db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 924.666062] env[63593]: DEBUG nova.network.neutron [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 924.677229] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367989, 'name': CreateVM_Task, 'duration_secs': 0.281912} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 924.678014] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b053676a-81b8-4345-b257-1d67976fd077] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 924.678691] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 924.678857] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 924.679213] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 924.679684] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d4889975-a105-4c0a-9061-2a9a41aede24 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 924.683997] env[63593]: DEBUG oslo_vmware.api [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for the task: (returnval){ [ 924.683997] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5232c3e8-b719-55da-b2b8-c260d640242e" [ 924.683997] env[63593]: _type = "Task" [ 924.683997] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 924.692815] env[63593]: DEBUG oslo_vmware.api [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5232c3e8-b719-55da-b2b8-c260d640242e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 925.193738] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 925.194056] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 925.194241] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 925.209309] env[63593]: DEBUG nova.network.neutron [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 925.360965] env[63593]: DEBUG nova.network.neutron [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Updating instance_info_cache with network_info: [{"id": "bad74310-1723-4f07-b2ed-798288e20247", "address": "fa:16:3e:45:a7:26", "network": {"id": "db64afcc-fe87-4fa6-a298-dfe62a79d370", "bridge": "br-int", "label": "tempest-ServerActionsTestJSON-324024816-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "7a529e687cb043acae6837e7f4f25a25", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "8f441782-e89c-4815-b53e-af83c5d27902", "external-id": "nsx-vlan-transportzone-562", "segmentation_id": 562, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbad74310-17", "ovs_interfaceid": "bad74310-1723-4f07-b2ed-798288e20247", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 925.863256] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Releasing lock "refresh_cache-cf724d13-e34e-4a17-9b54-56190891d5db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 925.863855] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Instance network_info: |[{"id": "bad74310-1723-4f07-b2ed-798288e20247", "address": "fa:16:3e:45:a7:26", "network": {"id": "db64afcc-fe87-4fa6-a298-dfe62a79d370", "bridge": "br-int", "label": "tempest-ServerActionsTestJSON-324024816-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "7a529e687cb043acae6837e7f4f25a25", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "8f441782-e89c-4815-b53e-af83c5d27902", "external-id": "nsx-vlan-transportzone-562", "segmentation_id": 562, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbad74310-17", "ovs_interfaceid": "bad74310-1723-4f07-b2ed-798288e20247", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 925.864363] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:45:a7:26', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '8f441782-e89c-4815-b53e-af83c5d27902', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'bad74310-1723-4f07-b2ed-798288e20247', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 925.871729] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Creating folder: Project (7a529e687cb043acae6837e7f4f25a25). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 925.872016] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-4b1f469e-ec64-4f73-8bed-2936c3469a84 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.883961] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Created folder: Project (7a529e687cb043acae6837e7f4f25a25) in parent group-v291016. [ 925.883961] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Creating folder: Instances. Parent ref: group-v291057. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 925.883961] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d20d6ce1-ae03-4c82-a472-0c0dc5f4c4e4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.891886] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Created folder: Instances in parent group-v291057. [ 925.892123] env[63593]: DEBUG oslo.service.loopingcall [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 925.892305] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 925.892890] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-ae216a32-a3c1-4338-9c08-9e0eb731f39c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.913048] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 925.913048] env[63593]: value = "task-1367992" [ 925.913048] env[63593]: _type = "Task" [ 925.913048] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 925.920413] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367992, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 926.140782] env[63593]: DEBUG nova.compute.manager [req-5646c87e-f7c9-4283-9b93-c87b7e220f59 req-33444c55-07df-496a-9a9b-cb389af13b28 service nova] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Received event network-changed-bad74310-1723-4f07-b2ed-798288e20247 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 926.140976] env[63593]: DEBUG nova.compute.manager [req-5646c87e-f7c9-4283-9b93-c87b7e220f59 req-33444c55-07df-496a-9a9b-cb389af13b28 service nova] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Refreshing instance network info cache due to event network-changed-bad74310-1723-4f07-b2ed-798288e20247. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 926.141202] env[63593]: DEBUG oslo_concurrency.lockutils [req-5646c87e-f7c9-4283-9b93-c87b7e220f59 req-33444c55-07df-496a-9a9b-cb389af13b28 service nova] Acquiring lock "refresh_cache-cf724d13-e34e-4a17-9b54-56190891d5db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 926.141352] env[63593]: DEBUG oslo_concurrency.lockutils [req-5646c87e-f7c9-4283-9b93-c87b7e220f59 req-33444c55-07df-496a-9a9b-cb389af13b28 service nova] Acquired lock "refresh_cache-cf724d13-e34e-4a17-9b54-56190891d5db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 926.141502] env[63593]: DEBUG nova.network.neutron [req-5646c87e-f7c9-4283-9b93-c87b7e220f59 req-33444c55-07df-496a-9a9b-cb389af13b28 service nova] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Refreshing network info cache for port bad74310-1723-4f07-b2ed-798288e20247 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 926.422798] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367992, 'name': CreateVM_Task, 'duration_secs': 0.299472} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 926.423158] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 926.423636] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 926.423799] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 926.424117] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 926.424360] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-66c6c4af-c4a1-41a5-acc3-486c219b8165 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 926.428908] env[63593]: DEBUG oslo_vmware.api [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Waiting for the task: (returnval){ [ 926.428908] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52f54003-fbe9-3fd5-e64d-ddc8291f94cf" [ 926.428908] env[63593]: _type = "Task" [ 926.428908] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 926.436099] env[63593]: DEBUG oslo_vmware.api [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52f54003-fbe9-3fd5-e64d-ddc8291f94cf, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 926.939176] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 926.939438] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 926.939642] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 926.965448] env[63593]: DEBUG nova.network.neutron [req-5646c87e-f7c9-4283-9b93-c87b7e220f59 req-33444c55-07df-496a-9a9b-cb389af13b28 service nova] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Updated VIF entry in instance network info cache for port bad74310-1723-4f07-b2ed-798288e20247. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 926.965965] env[63593]: DEBUG nova.network.neutron [req-5646c87e-f7c9-4283-9b93-c87b7e220f59 req-33444c55-07df-496a-9a9b-cb389af13b28 service nova] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Updating instance_info_cache with network_info: [{"id": "bad74310-1723-4f07-b2ed-798288e20247", "address": "fa:16:3e:45:a7:26", "network": {"id": "db64afcc-fe87-4fa6-a298-dfe62a79d370", "bridge": "br-int", "label": "tempest-ServerActionsTestJSON-324024816-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "7a529e687cb043acae6837e7f4f25a25", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "8f441782-e89c-4815-b53e-af83c5d27902", "external-id": "nsx-vlan-transportzone-562", "segmentation_id": 562, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbad74310-17", "ovs_interfaceid": "bad74310-1723-4f07-b2ed-798288e20247", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 927.469364] env[63593]: DEBUG oslo_concurrency.lockutils [req-5646c87e-f7c9-4283-9b93-c87b7e220f59 req-33444c55-07df-496a-9a9b-cb389af13b28 service nova] Releasing lock "refresh_cache-cf724d13-e34e-4a17-9b54-56190891d5db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 948.871453] env[63593]: WARNING oslo_vmware.rw_handles [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 948.871453] env[63593]: ERROR oslo_vmware.rw_handles [ 948.872098] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/10abfe60-f21d-49b2-ab2b-822ce0f0d754/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 948.874592] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 948.874831] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Copying Virtual Disk [datastore2] vmware_temp/10abfe60-f21d-49b2-ab2b-822ce0f0d754/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/10abfe60-f21d-49b2-ab2b-822ce0f0d754/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 948.875126] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-0ccc5642-28b1-4712-b000-886250a1c062 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.883198] env[63593]: DEBUG oslo_vmware.api [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for the task: (returnval){ [ 948.883198] env[63593]: value = "task-1367993" [ 948.883198] env[63593]: _type = "Task" [ 948.883198] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 948.890971] env[63593]: DEBUG oslo_vmware.api [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': task-1367993, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 949.407052] env[63593]: DEBUG oslo_vmware.exceptions [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 949.407240] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 949.407792] env[63593]: ERROR nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 949.407792] env[63593]: Faults: ['InvalidArgument'] [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Traceback (most recent call last): [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] yield resources [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] self.driver.spawn(context, instance, image_meta, [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] self._vmops.spawn(context, instance, image_meta, injected_files, [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] self._fetch_image_if_missing(context, vi) [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] image_cache(vi, tmp_image_ds_loc) [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] vm_util.copy_virtual_disk( [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] session._wait_for_task(vmdk_copy_task) [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] return self.wait_for_task(task_ref) [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] return evt.wait() [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] result = hub.switch() [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] return self.greenlet.switch() [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] self.f(*self.args, **self.kw) [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] raise exceptions.translate_fault(task_info.error) [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Faults: ['InvalidArgument'] [ 949.407792] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] [ 949.408761] env[63593]: INFO nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Terminating instance [ 949.409744] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 949.410642] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 949.410642] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-15dfb7aa-02c1-4121-b065-8ffd8a2bf73a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.412412] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "refresh_cache-4e312fd5-01bf-4bd4-8a61-26624ae05036" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 949.412531] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired lock "refresh_cache-4e312fd5-01bf-4bd4-8a61-26624ae05036" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 949.412690] env[63593]: DEBUG nova.network.neutron [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 949.419996] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 949.420193] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 949.421481] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-07f93f0b-f5c7-4be8-85f8-631cb6342928 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.427216] env[63593]: DEBUG oslo_vmware.api [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Waiting for the task: (returnval){ [ 949.427216] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]522b40ab-b597-4ba7-0c5a-c0167d8bf44a" [ 949.427216] env[63593]: _type = "Task" [ 949.427216] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 949.436035] env[63593]: DEBUG oslo_vmware.api [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]522b40ab-b597-4ba7-0c5a-c0167d8bf44a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 949.933827] env[63593]: DEBUG nova.network.neutron [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 949.941589] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 949.941904] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Creating directory with path [datastore2] vmware_temp/00bae9ca-d9aa-482a-9059-34fec2c2ebdd/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 949.942201] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-3b2c6a2c-6a41-4420-b2d5-20fa409ef1ce {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.961454] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Created directory with path [datastore2] vmware_temp/00bae9ca-d9aa-482a-9059-34fec2c2ebdd/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 949.961658] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Fetch image to [datastore2] vmware_temp/00bae9ca-d9aa-482a-9059-34fec2c2ebdd/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 949.961823] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/00bae9ca-d9aa-482a-9059-34fec2c2ebdd/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 949.962599] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfbd6959-afc3-49ff-ae9c-af14c6c1c6d3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.969341] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4112df05-c827-419c-a109-a9edcfbbb659 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.979063] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30e963e0-7ba5-4209-89e7-7e6202b5964e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.982983] env[63593]: DEBUG nova.network.neutron [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 950.010451] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1efbb0a8-42f9-45cd-a2e9-b94b5d7e04f6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.015960] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-8ec021a4-0cf0-4a90-838c-e99fbbe68e06 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.036029] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 950.083275] env[63593]: DEBUG oslo_vmware.rw_handles [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/00bae9ca-d9aa-482a-9059-34fec2c2ebdd/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 950.145050] env[63593]: DEBUG oslo_vmware.rw_handles [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 950.145303] env[63593]: DEBUG oslo_vmware.rw_handles [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/00bae9ca-d9aa-482a-9059-34fec2c2ebdd/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 950.474120] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.474368] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.485485] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Releasing lock "refresh_cache-4e312fd5-01bf-4bd4-8a61-26624ae05036" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 950.485900] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 950.486100] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 950.486930] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-609514aa-b643-485f-b761-b2717b4c58ed {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.495180] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 950.495800] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-436e58ee-898b-46a5-960f-9d8674c452bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.526622] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 950.526844] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 950.527032] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Deleting the datastore file [datastore2] 4e312fd5-01bf-4bd4-8a61-26624ae05036 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 950.527291] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-19d7b3e6-55dc-44d4-836e-2097cdd99666 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.533772] env[63593]: DEBUG oslo_vmware.api [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for the task: (returnval){ [ 950.533772] env[63593]: value = "task-1367995" [ 950.533772] env[63593]: _type = "Task" [ 950.533772] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 950.541199] env[63593]: DEBUG oslo_vmware.api [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': task-1367995, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 950.981725] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.981725] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 950.981725] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 951.043134] env[63593]: DEBUG oslo_vmware.api [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Task: {'id': task-1367995, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.033042} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 951.043400] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 951.043584] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 951.043753] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 951.043921] env[63593]: INFO nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Took 0.56 seconds to destroy the instance on the hypervisor. [ 951.044237] env[63593]: DEBUG oslo.service.loopingcall [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 951.044375] env[63593]: DEBUG nova.compute.manager [-] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 951.046452] env[63593]: DEBUG nova.compute.claims [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 951.046620] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 951.046826] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 951.485502] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.485751] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.485889] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.486026] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.486155] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.486276] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.486399] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.486519] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.486637] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b053676a-81b8-4345-b257-1d67976fd077] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.486755] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 951.486872] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 951.487091] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 951.487256] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 951.487403] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 951.487550] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 951.487692] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 951.487835] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 951.487960] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 951.488119] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 951.776617] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dabe1d98-8874-49ac-8e88-96d8e6508b04 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.784266] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e1c8080-7505-4bd1-b62d-65307eccffb0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.814217] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-728752d9-00c4-4433-a52c-094d8d6cfa6e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.821211] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91916c13-fe88-4bb4-9267-91ba0254cd40 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.834261] env[63593]: DEBUG nova.compute.provider_tree [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 951.991270] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 952.337537] env[63593]: DEBUG nova.scheduler.client.report [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 952.843135] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.796s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 952.843516] env[63593]: ERROR nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 952.843516] env[63593]: Faults: ['InvalidArgument'] [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Traceback (most recent call last): [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] self.driver.spawn(context, instance, image_meta, [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] self._vmops.spawn(context, instance, image_meta, injected_files, [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] self._fetch_image_if_missing(context, vi) [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] image_cache(vi, tmp_image_ds_loc) [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] vm_util.copy_virtual_disk( [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] session._wait_for_task(vmdk_copy_task) [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] return self.wait_for_task(task_ref) [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] return evt.wait() [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] result = hub.switch() [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] return self.greenlet.switch() [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] self.f(*self.args, **self.kw) [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] raise exceptions.translate_fault(task_info.error) [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Faults: ['InvalidArgument'] [ 952.843516] env[63593]: ERROR nova.compute.manager [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] [ 952.844279] env[63593]: DEBUG nova.compute.utils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 952.845398] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.854s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 952.845601] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 952.845776] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 952.846608] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca410cc4-776c-4ede-88a1-d26a5e13409d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.850476] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Build of instance 4e312fd5-01bf-4bd4-8a61-26624ae05036 was re-scheduled: A specified parameter was not correct: fileType [ 952.850476] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 952.851143] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 952.851143] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "refresh_cache-4e312fd5-01bf-4bd4-8a61-26624ae05036" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 952.851293] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired lock "refresh_cache-4e312fd5-01bf-4bd4-8a61-26624ae05036" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 952.851486] env[63593]: DEBUG nova.network.neutron [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 952.857582] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b2c011f-8c97-40de-a19a-9ee9eab43883 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.873257] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01267c25-bd40-41b8-abc8-5062070245de {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.879630] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f93f4a25-48c0-4704-80d5-258dd001ed73 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.908339] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181453MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 952.908528] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 952.908730] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 953.371376] env[63593]: DEBUG nova.network.neutron [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 953.417724] env[63593]: DEBUG nova.network.neutron [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.923932] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Releasing lock "refresh_cache-4e312fd5-01bf-4bd4-8a61-26624ae05036" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 953.924241] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 953.924364] env[63593]: DEBUG nova.compute.manager [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 954.444238] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e312fd5-01bf-4bd4-8a61-26624ae05036 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 954.444644] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 991481e2-557e-4222-adfc-d298fa9314a0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 954.444644] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a223dd01-d07d-4ac5-acfc-2e03b4c7dff4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 954.444644] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d3a4902f-1829-427c-9f37-6e83ae7265e3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 954.444771] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 954.444860] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fbd620a6-3743-4737-98b7-b13928b6587d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 954.444975] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 954.445105] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 328b91c6-4ae4-442f-ac87-519658a812ff actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 954.445217] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 954.445328] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 954.948529] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 954.952236] env[63593]: INFO nova.scheduler.client.report [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Deleted allocations for instance 4e312fd5-01bf-4bd4-8a61-26624ae05036 [ 955.451366] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 955.462827] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c976cf9d-405b-436b-9dfd-a10487e2f1a9 tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "4e312fd5-01bf-4bd4-8a61-26624ae05036" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 233.663s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 955.464058] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "4e312fd5-01bf-4bd4-8a61-26624ae05036" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 37.473s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 955.464288] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "4e312fd5-01bf-4bd4-8a61-26624ae05036-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 955.464494] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "4e312fd5-01bf-4bd4-8a61-26624ae05036-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 955.464656] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "4e312fd5-01bf-4bd4-8a61-26624ae05036-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 955.466904] env[63593]: INFO nova.compute.manager [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Terminating instance [ 955.468467] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquiring lock "refresh_cache-4e312fd5-01bf-4bd4-8a61-26624ae05036" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 955.468705] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Acquired lock "refresh_cache-4e312fd5-01bf-4bd4-8a61-26624ae05036" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 955.468773] env[63593]: DEBUG nova.network.neutron [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 955.955390] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d760b552-e2ff-46db-88eb-f5f58fc88239 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 955.967243] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 955.996787] env[63593]: DEBUG nova.network.neutron [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 956.088828] env[63593]: DEBUG nova.network.neutron [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 956.459803] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 257c613c-8613-42e5-b400-b9c352a151f0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 956.493938] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 956.592940] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Releasing lock "refresh_cache-4e312fd5-01bf-4bd4-8a61-26624ae05036" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 956.593391] env[63593]: DEBUG nova.compute.manager [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 956.593633] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 956.593950] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9d569ad2-628e-4715-958b-b7afbe4633a5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 956.602930] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd02e970-6d7e-4c84-9b7b-1de691722998 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 956.629162] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4e312fd5-01bf-4bd4-8a61-26624ae05036 could not be found. [ 956.629365] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 956.629553] env[63593]: INFO nova.compute.manager [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Took 0.04 seconds to destroy the instance on the hypervisor. [ 956.629793] env[63593]: DEBUG oslo.service.loopingcall [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 956.630016] env[63593]: DEBUG nova.compute.manager [-] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 956.630114] env[63593]: DEBUG nova.network.neutron [-] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 956.645282] env[63593]: DEBUG nova.network.neutron [-] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 956.963285] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 957.147613] env[63593]: DEBUG nova.network.neutron [-] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 957.466878] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0c9d6b46-e1f1-4dcc-b5bf-c6ca520d4e0f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 957.651066] env[63593]: INFO nova.compute.manager [-] [instance: 4e312fd5-01bf-4bd4-8a61-26624ae05036] Took 1.02 seconds to deallocate network for instance. [ 957.970075] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4dd65fb0-e079-4520-8809-dfd275d876f7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 958.472632] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a4c51198-1e5d-4a59-8a9b-a997872e372b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 958.678958] env[63593]: DEBUG oslo_concurrency.lockutils [None req-ffb0a097-8000-4d5f-a686-0e696d40e51f tempest-ServerShowV247Test-136192111 tempest-ServerShowV247Test-136192111-project-member] Lock "4e312fd5-01bf-4bd4-8a61-26624ae05036" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 3.215s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 958.977795] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0cdcc9d3-9963-49b5-8744-241b865335e2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 959.481023] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 38e38231-7b05-43c7-8de2-993a03ed4f57 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 959.481911] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 9 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 959.481911] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2240MB phys_disk=200GB used_disk=9GB total_vcpus=48 used_vcpus=9 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 959.716429] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3c81a60-ff44-4f34-9cfc-a4a6ff75f4d3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.724157] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6235db9b-5ea6-4f9c-8ee9-c654d0b32a19 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.753426] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2a06df8-23ee-4f3b-881c-f0b31961ce83 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.760969] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10fb228e-3d2b-498b-bd8e-1ae62b23bd89 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.773712] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 960.278674] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 960.785032] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 960.785032] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 7.876s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 960.785401] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 4.291s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 960.786700] env[63593]: INFO nova.compute.claims [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 962.028357] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2097a7b6-384d-438a-9675-b92278db39cf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 962.036678] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a225f8a7-350b-4b04-be74-fe5c035dfddc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 962.066268] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b48f7921-2bd3-4dad-a7f9-27bd8882c91f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 962.073474] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9216ce2-e375-4f2c-bfda-1be78258b9f1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 962.086376] env[63593]: DEBUG nova.compute.provider_tree [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 962.591039] env[63593]: DEBUG nova.scheduler.client.report [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 963.095835] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.310s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 963.096334] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 963.601839] env[63593]: DEBUG nova.compute.utils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 963.603148] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 963.603323] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 963.643128] env[63593]: DEBUG nova.policy [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '34dd65f3fb5f4bf0897ef824c564262e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ff6806624a684b0392c57c83ec81f089', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 963.941371] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Successfully created port: e461921f-86d4-4c68-add7-cfad02b3da8c {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 964.107460] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 965.117654] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 965.141708] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 965.141943] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 965.142109] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 965.142291] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 965.142433] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 965.142577] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 965.142777] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 965.142932] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 965.143108] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 965.143271] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 965.143439] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 965.144409] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-504f2a40-2cf2-4e85-9e56-8c456c020197 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.152462] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7affdd00-8c8b-491a-8da8-339df363267e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 965.480892] env[63593]: DEBUG nova.compute.manager [req-cb6754fd-9d83-440d-bf64-f852df9ad75f req-4a955bb6-08f9-4849-abc8-393980780c50 service nova] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Received event network-vif-plugged-e461921f-86d4-4c68-add7-cfad02b3da8c {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 965.481141] env[63593]: DEBUG oslo_concurrency.lockutils [req-cb6754fd-9d83-440d-bf64-f852df9ad75f req-4a955bb6-08f9-4849-abc8-393980780c50 service nova] Acquiring lock "48e001ae-cc04-4d95-be43-2b2b3556c6db-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 965.481351] env[63593]: DEBUG oslo_concurrency.lockutils [req-cb6754fd-9d83-440d-bf64-f852df9ad75f req-4a955bb6-08f9-4849-abc8-393980780c50 service nova] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 965.481517] env[63593]: DEBUG oslo_concurrency.lockutils [req-cb6754fd-9d83-440d-bf64-f852df9ad75f req-4a955bb6-08f9-4849-abc8-393980780c50 service nova] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 965.481685] env[63593]: DEBUG nova.compute.manager [req-cb6754fd-9d83-440d-bf64-f852df9ad75f req-4a955bb6-08f9-4849-abc8-393980780c50 service nova] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] No waiting events found dispatching network-vif-plugged-e461921f-86d4-4c68-add7-cfad02b3da8c {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 965.481849] env[63593]: WARNING nova.compute.manager [req-cb6754fd-9d83-440d-bf64-f852df9ad75f req-4a955bb6-08f9-4849-abc8-393980780c50 service nova] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Received unexpected event network-vif-plugged-e461921f-86d4-4c68-add7-cfad02b3da8c for instance with vm_state building and task_state spawning. [ 965.595411] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Successfully updated port: e461921f-86d4-4c68-add7-cfad02b3da8c {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 966.098484] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "refresh_cache-48e001ae-cc04-4d95-be43-2b2b3556c6db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 966.098678] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquired lock "refresh_cache-48e001ae-cc04-4d95-be43-2b2b3556c6db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 966.098863] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 966.631176] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 966.753163] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Updating instance_info_cache with network_info: [{"id": "e461921f-86d4-4c68-add7-cfad02b3da8c", "address": "fa:16:3e:64:ef:a7", "network": {"id": "84391498-e2ff-4de6-9dbf-e7af5f99795f", "bridge": "br-int", "label": "tempest-ListServersNegativeTestJSON-1479042547-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff6806624a684b0392c57c83ec81f089", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "023d6500-887e-4dc4-bec5-06b40450d9c0", "external-id": "nsx-vlan-transportzone-108", "segmentation_id": 108, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape461921f-86", "ovs_interfaceid": "e461921f-86d4-4c68-add7-cfad02b3da8c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 967.255889] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Releasing lock "refresh_cache-48e001ae-cc04-4d95-be43-2b2b3556c6db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 967.256260] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Instance network_info: |[{"id": "e461921f-86d4-4c68-add7-cfad02b3da8c", "address": "fa:16:3e:64:ef:a7", "network": {"id": "84391498-e2ff-4de6-9dbf-e7af5f99795f", "bridge": "br-int", "label": "tempest-ListServersNegativeTestJSON-1479042547-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff6806624a684b0392c57c83ec81f089", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "023d6500-887e-4dc4-bec5-06b40450d9c0", "external-id": "nsx-vlan-transportzone-108", "segmentation_id": 108, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape461921f-86", "ovs_interfaceid": "e461921f-86d4-4c68-add7-cfad02b3da8c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 967.256705] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:64:ef:a7', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '023d6500-887e-4dc4-bec5-06b40450d9c0', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'e461921f-86d4-4c68-add7-cfad02b3da8c', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 967.264253] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Creating folder: Project (ff6806624a684b0392c57c83ec81f089). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 967.264541] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b860b2f5-2359-4164-89ac-881a0780fb4d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.275358] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Created folder: Project (ff6806624a684b0392c57c83ec81f089) in parent group-v291016. [ 967.275529] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Creating folder: Instances. Parent ref: group-v291060. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 967.275738] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6abbcbff-4f9a-43f9-8829-8ed4a223a0d2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.285392] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Created folder: Instances in parent group-v291060. [ 967.285630] env[63593]: DEBUG oslo.service.loopingcall [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 967.285812] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 967.285993] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-490d602a-75da-4dda-9ac2-22a56da7230a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.303374] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 967.303374] env[63593]: value = "task-1367998" [ 967.303374] env[63593]: _type = "Task" [ 967.303374] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 967.310296] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367998, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 967.506296] env[63593]: DEBUG nova.compute.manager [req-58256e82-8a4a-4306-b1da-e14154c5855a req-053abf8b-6f44-48fa-a8fb-c376d2a0f10e service nova] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Received event network-changed-e461921f-86d4-4c68-add7-cfad02b3da8c {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 967.506507] env[63593]: DEBUG nova.compute.manager [req-58256e82-8a4a-4306-b1da-e14154c5855a req-053abf8b-6f44-48fa-a8fb-c376d2a0f10e service nova] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Refreshing instance network info cache due to event network-changed-e461921f-86d4-4c68-add7-cfad02b3da8c. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 967.506694] env[63593]: DEBUG oslo_concurrency.lockutils [req-58256e82-8a4a-4306-b1da-e14154c5855a req-053abf8b-6f44-48fa-a8fb-c376d2a0f10e service nova] Acquiring lock "refresh_cache-48e001ae-cc04-4d95-be43-2b2b3556c6db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 967.506835] env[63593]: DEBUG oslo_concurrency.lockutils [req-58256e82-8a4a-4306-b1da-e14154c5855a req-053abf8b-6f44-48fa-a8fb-c376d2a0f10e service nova] Acquired lock "refresh_cache-48e001ae-cc04-4d95-be43-2b2b3556c6db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 967.506992] env[63593]: DEBUG nova.network.neutron [req-58256e82-8a4a-4306-b1da-e14154c5855a req-053abf8b-6f44-48fa-a8fb-c376d2a0f10e service nova] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Refreshing network info cache for port e461921f-86d4-4c68-add7-cfad02b3da8c {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 967.813539] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1367998, 'name': CreateVM_Task, 'duration_secs': 0.341844} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 967.813862] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 967.814343] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 967.814508] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 967.814856] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 967.815107] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ddd64efb-1e53-4d38-9e0b-82bad8f8027d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.819397] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for the task: (returnval){ [ 967.819397] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52fd430b-dd02-7565-a431-0d9ef7351653" [ 967.819397] env[63593]: _type = "Task" [ 967.819397] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 967.826776] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52fd430b-dd02-7565-a431-0d9ef7351653, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 968.297561] env[63593]: DEBUG nova.network.neutron [req-58256e82-8a4a-4306-b1da-e14154c5855a req-053abf8b-6f44-48fa-a8fb-c376d2a0f10e service nova] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Updated VIF entry in instance network info cache for port e461921f-86d4-4c68-add7-cfad02b3da8c. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 968.297910] env[63593]: DEBUG nova.network.neutron [req-58256e82-8a4a-4306-b1da-e14154c5855a req-053abf8b-6f44-48fa-a8fb-c376d2a0f10e service nova] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Updating instance_info_cache with network_info: [{"id": "e461921f-86d4-4c68-add7-cfad02b3da8c", "address": "fa:16:3e:64:ef:a7", "network": {"id": "84391498-e2ff-4de6-9dbf-e7af5f99795f", "bridge": "br-int", "label": "tempest-ListServersNegativeTestJSON-1479042547-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff6806624a684b0392c57c83ec81f089", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "023d6500-887e-4dc4-bec5-06b40450d9c0", "external-id": "nsx-vlan-transportzone-108", "segmentation_id": 108, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape461921f-86", "ovs_interfaceid": "e461921f-86d4-4c68-add7-cfad02b3da8c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 968.329620] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 968.329822] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 968.330045] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 968.800735] env[63593]: DEBUG oslo_concurrency.lockutils [req-58256e82-8a4a-4306-b1da-e14154c5855a req-053abf8b-6f44-48fa-a8fb-c376d2a0f10e service nova] Releasing lock "refresh_cache-48e001ae-cc04-4d95-be43-2b2b3556c6db" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 981.547993] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "991481e2-557e-4222-adfc-d298fa9314a0" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 987.913586] env[63593]: DEBUG oslo_concurrency.lockutils [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquiring lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 988.643611] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "d3a4902f-1829-427c-9f37-6e83ae7265e3" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 988.742393] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.891657] env[63593]: WARNING oslo_vmware.rw_handles [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 998.891657] env[63593]: ERROR oslo_vmware.rw_handles [ 998.892665] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/00bae9ca-d9aa-482a-9059-34fec2c2ebdd/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 998.893969] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 998.894223] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Copying Virtual Disk [datastore2] vmware_temp/00bae9ca-d9aa-482a-9059-34fec2c2ebdd/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/00bae9ca-d9aa-482a-9059-34fec2c2ebdd/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 998.894497] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-560a877e-6f1f-4872-9d84-78798550f040 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.903297] env[63593]: DEBUG oslo_vmware.api [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Waiting for the task: (returnval){ [ 998.903297] env[63593]: value = "task-1367999" [ 998.903297] env[63593]: _type = "Task" [ 998.903297] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 998.911182] env[63593]: DEBUG oslo_vmware.api [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Task: {'id': task-1367999, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 999.413534] env[63593]: DEBUG oslo_vmware.exceptions [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 999.413811] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 999.414409] env[63593]: ERROR nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 999.414409] env[63593]: Faults: ['InvalidArgument'] [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Traceback (most recent call last): [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] yield resources [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] self.driver.spawn(context, instance, image_meta, [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] self._fetch_image_if_missing(context, vi) [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] image_cache(vi, tmp_image_ds_loc) [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] vm_util.copy_virtual_disk( [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] session._wait_for_task(vmdk_copy_task) [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] return self.wait_for_task(task_ref) [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] return evt.wait() [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] result = hub.switch() [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] return self.greenlet.switch() [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] self.f(*self.args, **self.kw) [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] raise exceptions.translate_fault(task_info.error) [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Faults: ['InvalidArgument'] [ 999.414409] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] [ 999.415254] env[63593]: INFO nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Terminating instance [ 999.416247] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 999.416458] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 999.416689] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-afd7fcad-b117-46c1-817e-f07a99e1db44 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.418934] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "refresh_cache-991481e2-557e-4222-adfc-d298fa9314a0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 999.419109] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquired lock "refresh_cache-991481e2-557e-4222-adfc-d298fa9314a0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 999.419273] env[63593]: DEBUG nova.network.neutron [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 999.425771] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 999.426296] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 999.427081] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d453d3a8-7e62-4e80-b47b-48568faeb918 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.432141] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for the task: (returnval){ [ 999.432141] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52a40cdb-3a35-6d03-083e-d137e96ac253" [ 999.432141] env[63593]: _type = "Task" [ 999.432141] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 999.439341] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52a40cdb-3a35-6d03-083e-d137e96ac253, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 999.939132] env[63593]: DEBUG nova.network.neutron [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 999.944652] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 999.944900] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Creating directory with path [datastore2] vmware_temp/a49a90c6-4f33-4426-b7cf-97689b3e4750/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 999.945214] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ab733903-2d7f-4f6c-9bb6-9552b8786a8f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.967851] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Created directory with path [datastore2] vmware_temp/a49a90c6-4f33-4426-b7cf-97689b3e4750/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 999.968063] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Fetch image to [datastore2] vmware_temp/a49a90c6-4f33-4426-b7cf-97689b3e4750/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 999.968206] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/a49a90c6-4f33-4426-b7cf-97689b3e4750/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 999.968949] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f88243ca-251a-4aba-8206-aa807b7775a9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.975605] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-827e70f9-f363-49d5-8f4e-31fadf0f1048 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.984265] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e191bb34-8ff9-4aac-b109-0fd861e1e1bd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.014861] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2fb7e635-f0ba-4706-9d22-31fa202f2854 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.020543] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-a9c0f7cb-0a8c-4f0c-8f83-1df19a2b511c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.026705] env[63593]: DEBUG nova.network.neutron [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1000.039517] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1000.087444] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/a49a90c6-4f33-4426-b7cf-97689b3e4750/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1000.150291] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1000.150507] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/a49a90c6-4f33-4426-b7cf-97689b3e4750/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1000.530066] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Releasing lock "refresh_cache-991481e2-557e-4222-adfc-d298fa9314a0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1000.530484] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1000.530686] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1000.531621] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-daf5f3a9-417f-411b-8163-b20a142c4dfb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.539606] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1000.539828] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-d4210036-462e-4ee6-a43b-963b879bf221 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.567087] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1000.567291] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1000.567465] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Deleting the datastore file [datastore2] 991481e2-557e-4222-adfc-d298fa9314a0 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1000.567689] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0ade2aeb-ff4c-4795-a369-3d0b2f0ad76b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.573973] env[63593]: DEBUG oslo_vmware.api [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Waiting for the task: (returnval){ [ 1000.573973] env[63593]: value = "task-1368001" [ 1000.573973] env[63593]: _type = "Task" [ 1000.573973] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1000.581844] env[63593]: DEBUG oslo_vmware.api [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Task: {'id': task-1368001, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1001.084353] env[63593]: DEBUG oslo_vmware.api [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Task: {'id': task-1368001, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.033493} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1001.084649] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1001.084785] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1001.084954] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1001.085142] env[63593]: INFO nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Took 0.55 seconds to destroy the instance on the hypervisor. [ 1001.085382] env[63593]: DEBUG oslo.service.loopingcall [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1001.085696] env[63593]: DEBUG nova.compute.manager [-] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 1001.087919] env[63593]: DEBUG nova.compute.claims [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1001.088139] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1001.088361] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1001.796597] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e878f59-89cb-4004-b922-35c86c3dee54 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.804192] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d9d9dfe-8622-49cc-9af2-49dafed61af0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.833647] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77ea7c9c-84ee-49a6-91bd-6a517c04bf7c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.841116] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93c7c6d1-b5fe-439f-8a98-26304b59e0f4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.855206] env[63593]: DEBUG nova.compute.provider_tree [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1002.359692] env[63593]: DEBUG nova.scheduler.client.report [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1002.864039] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.775s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1002.864549] env[63593]: ERROR nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1002.864549] env[63593]: Faults: ['InvalidArgument'] [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Traceback (most recent call last): [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] self.driver.spawn(context, instance, image_meta, [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] self._fetch_image_if_missing(context, vi) [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] image_cache(vi, tmp_image_ds_loc) [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] vm_util.copy_virtual_disk( [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] session._wait_for_task(vmdk_copy_task) [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] return self.wait_for_task(task_ref) [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] return evt.wait() [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] result = hub.switch() [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] return self.greenlet.switch() [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] self.f(*self.args, **self.kw) [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] raise exceptions.translate_fault(task_info.error) [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Faults: ['InvalidArgument'] [ 1002.864549] env[63593]: ERROR nova.compute.manager [instance: 991481e2-557e-4222-adfc-d298fa9314a0] [ 1002.865368] env[63593]: DEBUG nova.compute.utils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1002.867142] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Build of instance 991481e2-557e-4222-adfc-d298fa9314a0 was re-scheduled: A specified parameter was not correct: fileType [ 1002.867142] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1002.867514] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1002.867764] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "refresh_cache-991481e2-557e-4222-adfc-d298fa9314a0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1002.867893] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquired lock "refresh_cache-991481e2-557e-4222-adfc-d298fa9314a0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1002.868063] env[63593]: DEBUG nova.network.neutron [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1003.386112] env[63593]: DEBUG nova.network.neutron [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1003.428271] env[63593]: DEBUG nova.network.neutron [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1003.931118] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Releasing lock "refresh_cache-991481e2-557e-4222-adfc-d298fa9314a0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1003.931384] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1003.931565] env[63593]: DEBUG nova.compute.manager [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 1004.958801] env[63593]: INFO nova.scheduler.client.report [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Deleted allocations for instance 991481e2-557e-4222-adfc-d298fa9314a0 [ 1005.466699] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3d4da7bf-2402-4b92-89cd-dae8d6a01867 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "991481e2-557e-4222-adfc-d298fa9314a0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 220.804s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1005.468080] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "991481e2-557e-4222-adfc-d298fa9314a0" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 23.920s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1005.468350] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "991481e2-557e-4222-adfc-d298fa9314a0-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1005.468598] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "991481e2-557e-4222-adfc-d298fa9314a0-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1005.468789] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "991481e2-557e-4222-adfc-d298fa9314a0-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1005.473818] env[63593]: INFO nova.compute.manager [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Terminating instance [ 1005.475358] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquiring lock "refresh_cache-991481e2-557e-4222-adfc-d298fa9314a0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1005.475497] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Acquired lock "refresh_cache-991481e2-557e-4222-adfc-d298fa9314a0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1005.475691] env[63593]: DEBUG nova.network.neutron [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1005.971663] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1005.993629] env[63593]: DEBUG nova.network.neutron [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1006.043129] env[63593]: DEBUG nova.network.neutron [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1006.494132] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1006.494467] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1006.496324] env[63593]: INFO nova.compute.claims [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1006.545735] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Releasing lock "refresh_cache-991481e2-557e-4222-adfc-d298fa9314a0" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1006.546217] env[63593]: DEBUG nova.compute.manager [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1006.546440] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1006.547038] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a907a1cd-e11d-4520-8031-9d738719fb10 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.559536] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75117bce-e060-4a16-aa1e-fe75de3dab25 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.597803] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 991481e2-557e-4222-adfc-d298fa9314a0 could not be found. [ 1006.598225] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1006.598694] env[63593]: INFO nova.compute.manager [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1006.598817] env[63593]: DEBUG oslo.service.loopingcall [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1006.599141] env[63593]: DEBUG nova.compute.manager [-] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1006.599292] env[63593]: DEBUG nova.network.neutron [-] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1006.628415] env[63593]: DEBUG nova.network.neutron [-] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1007.130703] env[63593]: DEBUG nova.network.neutron [-] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1007.633172] env[63593]: INFO nova.compute.manager [-] [instance: 991481e2-557e-4222-adfc-d298fa9314a0] Took 1.03 seconds to deallocate network for instance. [ 1007.784360] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19865c5e-990f-4e57-a8f8-418a410b59e5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.793350] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11b9e1f1-d7e8-431e-ae70-cf68e73aca9d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.828489] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec6a3a19-30ad-4099-bf67-51564707c59a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.836696] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dab82449-0d90-4cdb-86d4-df3d5445bbfc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1007.855727] env[63593]: DEBUG nova.compute.provider_tree [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1008.359101] env[63593]: DEBUG nova.scheduler.client.report [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1008.661302] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c6a25281-9079-4b24-af76-5658429c7004 tempest-ServerShowV257Test-121385003 tempest-ServerShowV257Test-121385003-project-member] Lock "991481e2-557e-4222-adfc-d298fa9314a0" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 3.193s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1008.867932] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.373s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1008.868626] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1009.374010] env[63593]: DEBUG nova.compute.utils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1009.375393] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1009.375566] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1009.426487] env[63593]: DEBUG nova.policy [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '34dd65f3fb5f4bf0897ef824c564262e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ff6806624a684b0392c57c83ec81f089', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1009.785011] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Successfully created port: 2b082ab3-9f60-437a-b552-d9d13bb2c9a1 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1009.878734] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1010.889295] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1010.911008] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1010.911273] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1010.911429] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1010.911607] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1010.911752] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1010.911896] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1010.912116] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1010.912285] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1010.912451] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1010.912610] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1010.912778] env[63593]: DEBUG nova.virt.hardware [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1010.913645] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba091a55-7108-41d4-b74d-ab16202c251a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1010.921657] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5836dade-cce8-4424-b72c-b507d0138052 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1011.572883] env[63593]: DEBUG nova.compute.manager [req-4603c9b2-d59b-4a2c-858b-cae60dcb9c81 req-3bd9627c-261b-48c5-a23d-d634151d061a service nova] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Received event network-vif-plugged-2b082ab3-9f60-437a-b552-d9d13bb2c9a1 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1011.573131] env[63593]: DEBUG oslo_concurrency.lockutils [req-4603c9b2-d59b-4a2c-858b-cae60dcb9c81 req-3bd9627c-261b-48c5-a23d-d634151d061a service nova] Acquiring lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1011.573347] env[63593]: DEBUG oslo_concurrency.lockutils [req-4603c9b2-d59b-4a2c-858b-cae60dcb9c81 req-3bd9627c-261b-48c5-a23d-d634151d061a service nova] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1011.573512] env[63593]: DEBUG oslo_concurrency.lockutils [req-4603c9b2-d59b-4a2c-858b-cae60dcb9c81 req-3bd9627c-261b-48c5-a23d-d634151d061a service nova] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1011.573676] env[63593]: DEBUG nova.compute.manager [req-4603c9b2-d59b-4a2c-858b-cae60dcb9c81 req-3bd9627c-261b-48c5-a23d-d634151d061a service nova] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] No waiting events found dispatching network-vif-plugged-2b082ab3-9f60-437a-b552-d9d13bb2c9a1 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1011.573853] env[63593]: WARNING nova.compute.manager [req-4603c9b2-d59b-4a2c-858b-cae60dcb9c81 req-3bd9627c-261b-48c5-a23d-d634151d061a service nova] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Received unexpected event network-vif-plugged-2b082ab3-9f60-437a-b552-d9d13bb2c9a1 for instance with vm_state building and task_state spawning. [ 1011.623741] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Successfully updated port: 2b082ab3-9f60-437a-b552-d9d13bb2c9a1 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1011.686842] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquiring lock "488c4bca-8279-4de7-8259-238497cbd011" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1011.686842] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "488c4bca-8279-4de7-8259-238497cbd011" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1012.128934] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "refresh_cache-6a21b30c-1e56-4c0c-ad7d-e0987f70d203" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1012.129197] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquired lock "refresh_cache-6a21b30c-1e56-4c0c-ad7d-e0987f70d203" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1012.129376] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1012.662513] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1012.826859] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Updating instance_info_cache with network_info: [{"id": "2b082ab3-9f60-437a-b552-d9d13bb2c9a1", "address": "fa:16:3e:93:3e:da", "network": {"id": "84391498-e2ff-4de6-9dbf-e7af5f99795f", "bridge": "br-int", "label": "tempest-ListServersNegativeTestJSON-1479042547-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff6806624a684b0392c57c83ec81f089", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "023d6500-887e-4dc4-bec5-06b40450d9c0", "external-id": "nsx-vlan-transportzone-108", "segmentation_id": 108, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2b082ab3-9f", "ovs_interfaceid": "2b082ab3-9f60-437a-b552-d9d13bb2c9a1", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1013.331081] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Releasing lock "refresh_cache-6a21b30c-1e56-4c0c-ad7d-e0987f70d203" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1013.331081] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Instance network_info: |[{"id": "2b082ab3-9f60-437a-b552-d9d13bb2c9a1", "address": "fa:16:3e:93:3e:da", "network": {"id": "84391498-e2ff-4de6-9dbf-e7af5f99795f", "bridge": "br-int", "label": "tempest-ListServersNegativeTestJSON-1479042547-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff6806624a684b0392c57c83ec81f089", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "023d6500-887e-4dc4-bec5-06b40450d9c0", "external-id": "nsx-vlan-transportzone-108", "segmentation_id": 108, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2b082ab3-9f", "ovs_interfaceid": "2b082ab3-9f60-437a-b552-d9d13bb2c9a1", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1013.331386] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:93:3e:da', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '023d6500-887e-4dc4-bec5-06b40450d9c0', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '2b082ab3-9f60-437a-b552-d9d13bb2c9a1', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1013.338745] env[63593]: DEBUG oslo.service.loopingcall [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1013.338956] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1013.339201] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-31b48205-9d51-4b23-aadb-55c7ec94bccc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.358350] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1013.358350] env[63593]: value = "task-1368002" [ 1013.358350] env[63593]: _type = "Task" [ 1013.358350] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1013.365609] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368002, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1013.676904] env[63593]: DEBUG nova.compute.manager [req-63d529d6-a361-459f-84e2-b9ec1336342b req-e5666fe0-ffc4-4f90-b880-e8319dbcf9fb service nova] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Received event network-changed-2b082ab3-9f60-437a-b552-d9d13bb2c9a1 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1013.677117] env[63593]: DEBUG nova.compute.manager [req-63d529d6-a361-459f-84e2-b9ec1336342b req-e5666fe0-ffc4-4f90-b880-e8319dbcf9fb service nova] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Refreshing instance network info cache due to event network-changed-2b082ab3-9f60-437a-b552-d9d13bb2c9a1. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1013.677535] env[63593]: DEBUG oslo_concurrency.lockutils [req-63d529d6-a361-459f-84e2-b9ec1336342b req-e5666fe0-ffc4-4f90-b880-e8319dbcf9fb service nova] Acquiring lock "refresh_cache-6a21b30c-1e56-4c0c-ad7d-e0987f70d203" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1013.677535] env[63593]: DEBUG oslo_concurrency.lockutils [req-63d529d6-a361-459f-84e2-b9ec1336342b req-e5666fe0-ffc4-4f90-b880-e8319dbcf9fb service nova] Acquired lock "refresh_cache-6a21b30c-1e56-4c0c-ad7d-e0987f70d203" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1013.677613] env[63593]: DEBUG nova.network.neutron [req-63d529d6-a361-459f-84e2-b9ec1336342b req-e5666fe0-ffc4-4f90-b880-e8319dbcf9fb service nova] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Refreshing network info cache for port 2b082ab3-9f60-437a-b552-d9d13bb2c9a1 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1013.868536] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368002, 'name': CreateVM_Task, 'duration_secs': 0.311391} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1013.868708] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1013.869389] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1013.869554] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1013.869866] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1013.870139] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8ce9a95d-7aff-426c-919f-481792a51468 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1013.874815] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for the task: (returnval){ [ 1013.874815] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5219e175-7b08-45b9-f650-3a77ffa949a1" [ 1013.874815] env[63593]: _type = "Task" [ 1013.874815] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1013.882107] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5219e175-7b08-45b9-f650-3a77ffa949a1, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1014.379917] env[63593]: DEBUG nova.network.neutron [req-63d529d6-a361-459f-84e2-b9ec1336342b req-e5666fe0-ffc4-4f90-b880-e8319dbcf9fb service nova] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Updated VIF entry in instance network info cache for port 2b082ab3-9f60-437a-b552-d9d13bb2c9a1. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1014.380309] env[63593]: DEBUG nova.network.neutron [req-63d529d6-a361-459f-84e2-b9ec1336342b req-e5666fe0-ffc4-4f90-b880-e8319dbcf9fb service nova] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Updating instance_info_cache with network_info: [{"id": "2b082ab3-9f60-437a-b552-d9d13bb2c9a1", "address": "fa:16:3e:93:3e:da", "network": {"id": "84391498-e2ff-4de6-9dbf-e7af5f99795f", "bridge": "br-int", "label": "tempest-ListServersNegativeTestJSON-1479042547-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "ff6806624a684b0392c57c83ec81f089", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "023d6500-887e-4dc4-bec5-06b40450d9c0", "external-id": "nsx-vlan-transportzone-108", "segmentation_id": 108, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2b082ab3-9f", "ovs_interfaceid": "2b082ab3-9f60-437a-b552-d9d13bb2c9a1", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1014.387131] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1014.387364] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1014.387570] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1014.883522] env[63593]: DEBUG oslo_concurrency.lockutils [req-63d529d6-a361-459f-84e2-b9ec1336342b req-e5666fe0-ffc4-4f90-b880-e8319dbcf9fb service nova] Releasing lock "refresh_cache-6a21b30c-1e56-4c0c-ad7d-e0987f70d203" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1016.429757] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "fbd620a6-3743-4737-98b7-b13928b6587d" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1019.714540] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1020.042014] env[63593]: DEBUG oslo_concurrency.lockutils [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "328b91c6-4ae4-442f-ac87-519658a812ff" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1020.790575] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1020.790875] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1020.790875] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1020.790875] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1021.295576] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.295793] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.295872] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.295998] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.296144] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.296269] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.296390] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b053676a-81b8-4345-b257-1d67976fd077] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.296510] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.296628] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.296746] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1021.296997] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1021.297074] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1021.297213] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1021.297359] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1021.297514] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1021.297723] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1021.297921] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1021.298087] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1021.298263] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1021.801692] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1021.802052] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1021.802105] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1021.802262] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1021.803155] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3634f839-d637-4397-8d39-bb51235b4cf0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1021.811364] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4eb430e2-a86e-4320-8dc0-31e9a75c9725 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1021.824915] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8013e132-a579-4d04-825c-2c165da2c650 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1021.831015] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b90143f-fc82-411b-88b4-cf74fea1953f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1021.860153] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181450MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1021.860314] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1021.860504] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1022.894027] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a223dd01-d07d-4ac5-acfc-2e03b4c7dff4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1022.894027] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d3a4902f-1829-427c-9f37-6e83ae7265e3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1022.894027] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1022.894402] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fbd620a6-3743-4737-98b7-b13928b6587d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1022.894402] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1022.894402] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 328b91c6-4ae4-442f-ac87-519658a812ff actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1022.894500] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1022.894580] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1022.894688] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1022.894795] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1023.398314] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d760b552-e2ff-46db-88eb-f5f58fc88239 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1023.902023] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 257c613c-8613-42e5-b400-b9c352a151f0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1024.405309] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1024.908938] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0c9d6b46-e1f1-4dcc-b5bf-c6ca520d4e0f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1025.411292] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4dd65fb0-e079-4520-8809-dfd275d876f7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1025.711462] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "b053676a-81b8-4345-b257-1d67976fd077" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1025.914875] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a4c51198-1e5d-4a59-8a9b-a997872e372b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1026.417636] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0cdcc9d3-9963-49b5-8744-241b865335e2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1026.920102] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 38e38231-7b05-43c7-8de2-993a03ed4f57 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1027.163451] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1027.172061] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquiring lock "cf724d13-e34e-4a17-9b54-56190891d5db" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1027.227239] env[63593]: DEBUG oslo_concurrency.lockutils [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "48e001ae-cc04-4d95-be43-2b2b3556c6db" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1027.423125] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1027.423406] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1027.423558] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1027.623988] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bebd2b30-cb06-486c-a8c6-d461a52eadb5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.632950] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbf5d907-e159-4ae9-80c8-b254990ffa80 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.662831] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-daa809ed-787a-44eb-a792-05a5062f130d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.669841] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89104740-694c-4ebd-92e5-f1bec4eb03c7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1027.682301] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1028.185374] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1028.692844] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1028.693074] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 6.833s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1034.348597] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquiring lock "a55c8243-3413-435e-89cb-6a2bb65b5292" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1034.348861] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1040.912635] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1040.913019] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.420035] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.420035] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1041.420248] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1041.924939] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.925245] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.925245] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.925397] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.925550] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.925672] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.925791] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b053676a-81b8-4345-b257-1d67976fd077] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.925910] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.926108] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.926246] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1041.926368] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1041.926565] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.926715] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.926859] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.927010] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.927162] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.927302] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.927427] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1041.927562] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1042.431826] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1042.432091] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1042.432261] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1042.432419] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1042.433360] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d9679c6-8858-4d30-919b-f5a725bf3141 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.441936] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4413b1e0-c03b-47aa-a14a-f070db020742 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.457320] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-721a95b3-58e8-400b-a612-2c227f72723a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.463703] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e79459c0-6b54-4d12-a1d3-51d47a94d178 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.492411] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181450MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1042.492570] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1042.492789] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1043.526276] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a223dd01-d07d-4ac5-acfc-2e03b4c7dff4 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1043.526702] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance d3a4902f-1829-427c-9f37-6e83ae7265e3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1043.526702] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1043.526702] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fbd620a6-3743-4737-98b7-b13928b6587d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1043.526884] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1043.526884] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 328b91c6-4ae4-442f-ac87-519658a812ff actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1043.527020] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1043.527106] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1043.527202] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1043.527317] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1044.030175] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1044.533848] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0c9d6b46-e1f1-4dcc-b5bf-c6ca520d4e0f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1045.038218] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4dd65fb0-e079-4520-8809-dfd275d876f7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1045.543937] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a4c51198-1e5d-4a59-8a9b-a997872e372b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1046.046596] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0cdcc9d3-9963-49b5-8744-241b865335e2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1046.550060] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 38e38231-7b05-43c7-8de2-993a03ed4f57 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1047.053927] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1047.556780] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1047.557105] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1047.557204] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1047.735668] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b45cbb6-9c51-42c6-9f20-0dc7571f44af {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.742957] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d08d342a-f6ad-46cd-8aae-72d60f1a9991 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.771320] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b02c78bf-f74a-4ef5-ae95-30fee457e230 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.778950] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ade696cf-6dde-49a7-a54f-a19110bc5d43 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1047.791919] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1048.295192] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1048.580738] env[63593]: WARNING oslo_vmware.rw_handles [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1048.580738] env[63593]: ERROR oslo_vmware.rw_handles [ 1048.581350] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/a49a90c6-4f33-4426-b7cf-97689b3e4750/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1048.583171] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1048.583421] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Copying Virtual Disk [datastore2] vmware_temp/a49a90c6-4f33-4426-b7cf-97689b3e4750/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/a49a90c6-4f33-4426-b7cf-97689b3e4750/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1048.583690] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-ea2d71c6-7a5a-4d0a-a3b3-5e3d6d8cd4e3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1048.592772] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for the task: (returnval){ [ 1048.592772] env[63593]: value = "task-1368003" [ 1048.592772] env[63593]: _type = "Task" [ 1048.592772] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1048.600333] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': task-1368003, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1048.800491] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1048.800652] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 6.308s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1049.103523] env[63593]: DEBUG oslo_vmware.exceptions [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1049.103826] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1049.104405] env[63593]: ERROR nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1049.104405] env[63593]: Faults: ['InvalidArgument'] [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Traceback (most recent call last): [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] yield resources [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] self.driver.spawn(context, instance, image_meta, [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] self._fetch_image_if_missing(context, vi) [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] image_cache(vi, tmp_image_ds_loc) [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] vm_util.copy_virtual_disk( [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] session._wait_for_task(vmdk_copy_task) [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] return self.wait_for_task(task_ref) [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] return evt.wait() [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] result = hub.switch() [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] return self.greenlet.switch() [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] self.f(*self.args, **self.kw) [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] raise exceptions.translate_fault(task_info.error) [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Faults: ['InvalidArgument'] [ 1049.104405] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] [ 1049.105351] env[63593]: INFO nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Terminating instance [ 1049.106931] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1049.106931] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1049.106931] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-912c820b-d5a8-4a96-b2ff-80686374a8a1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.108848] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1049.109049] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1049.109817] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6808110-bad8-4a7c-a708-c6552479906f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.116803] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1049.117047] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-cf607adf-67a5-4b88-8062-6df0a28e030b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.119192] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1049.119359] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1049.120262] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c5a16839-ac21-4fd8-bb3a-33bcb0a12814 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.125942] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for the task: (returnval){ [ 1049.125942] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52094451-dab4-72ae-0639-8b207f0fffba" [ 1049.125942] env[63593]: _type = "Task" [ 1049.125942] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1049.137334] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52094451-dab4-72ae-0639-8b207f0fffba, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1049.264982] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1049.265226] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1049.265404] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Deleting the datastore file [datastore2] a223dd01-d07d-4ac5-acfc-2e03b4c7dff4 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1049.265671] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-7112cb92-359a-4a45-a402-811c81272969 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.272197] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for the task: (returnval){ [ 1049.272197] env[63593]: value = "task-1368005" [ 1049.272197] env[63593]: _type = "Task" [ 1049.272197] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1049.279682] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': task-1368005, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1049.635542] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1049.635803] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Creating directory with path [datastore2] vmware_temp/52919336-80f9-4236-85a0-e4bb60e405ee/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1049.636046] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8a954902-8fa1-4bce-ad30-249713700640 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.647163] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Created directory with path [datastore2] vmware_temp/52919336-80f9-4236-85a0-e4bb60e405ee/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1049.647361] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Fetch image to [datastore2] vmware_temp/52919336-80f9-4236-85a0-e4bb60e405ee/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1049.647539] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/52919336-80f9-4236-85a0-e4bb60e405ee/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1049.648265] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-570d31d1-f2a1-44ba-b02d-96d403b3dc40 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.654664] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e7c3472-ec49-471b-9557-d4acac835d79 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.663311] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1bbbf30-36d7-4f09-998c-1cccc1f3ad38 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.693715] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9421ee1-3936-4e05-b1b2-cff21be60e2d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.698806] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-5dc5a299-08d8-4299-951b-72f7c96f31aa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1049.722406] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1049.769954] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/52919336-80f9-4236-85a0-e4bb60e405ee/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1049.833214] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': task-1368005, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.079849} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1049.834438] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1049.834624] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1049.834816] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1049.834980] env[63593]: INFO nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Took 0.73 seconds to destroy the instance on the hypervisor. [ 1049.836746] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1049.836932] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/52919336-80f9-4236-85a0-e4bb60e405ee/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1049.837500] env[63593]: DEBUG nova.compute.claims [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1049.837661] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1049.837867] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1050.536141] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc5bdc10-0af7-4b1e-97f7-83c1e34e23b3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1050.543408] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15be4439-7b52-4b86-9dfe-b43e62662209 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1050.572165] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-975e585a-a96b-492b-ae83-eb03fabbc285 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1050.579240] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcbf7754-c6e1-4dfd-81ea-b23bf47ffbd1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1050.592841] env[63593]: DEBUG nova.compute.provider_tree [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1051.097543] env[63593]: DEBUG nova.scheduler.client.report [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1051.602262] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.764s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1051.602833] env[63593]: ERROR nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1051.602833] env[63593]: Faults: ['InvalidArgument'] [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Traceback (most recent call last): [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] self.driver.spawn(context, instance, image_meta, [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] self._fetch_image_if_missing(context, vi) [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] image_cache(vi, tmp_image_ds_loc) [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] vm_util.copy_virtual_disk( [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] session._wait_for_task(vmdk_copy_task) [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] return self.wait_for_task(task_ref) [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] return evt.wait() [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] result = hub.switch() [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] return self.greenlet.switch() [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] self.f(*self.args, **self.kw) [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] raise exceptions.translate_fault(task_info.error) [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Faults: ['InvalidArgument'] [ 1051.602833] env[63593]: ERROR nova.compute.manager [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] [ 1051.604108] env[63593]: DEBUG nova.compute.utils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1051.605332] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Build of instance a223dd01-d07d-4ac5-acfc-2e03b4c7dff4 was re-scheduled: A specified parameter was not correct: fileType [ 1051.605332] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1051.605693] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1051.605860] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1051.606036] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1051.606200] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1052.372601] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1052.876740] env[63593]: INFO nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Took 1.27 seconds to deallocate network for instance. [ 1053.908977] env[63593]: INFO nova.scheduler.client.report [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Deleted allocations for instance a223dd01-d07d-4ac5-acfc-2e03b4c7dff4 [ 1054.420018] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 262.666s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1054.421427] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 65.679s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1054.421688] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1054.421906] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1054.422132] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1054.426728] env[63593]: INFO nova.compute.manager [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Terminating instance [ 1054.428882] env[63593]: DEBUG nova.compute.manager [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1054.429134] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1054.429443] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-1a8ba190-d473-49d6-90dd-c9e39d6c4c87 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1054.439090] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecd106b4-6388-4a1f-901a-acf77c31db77 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1054.466366] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a223dd01-d07d-4ac5-acfc-2e03b4c7dff4 could not be found. [ 1054.466626] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1054.466743] env[63593]: INFO nova.compute.manager [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1054.466959] env[63593]: DEBUG oslo.service.loopingcall [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1054.467237] env[63593]: DEBUG nova.compute.manager [-] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1054.467333] env[63593]: DEBUG nova.network.neutron [-] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1054.924558] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: d760b552-e2ff-46db-88eb-f5f58fc88239] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1054.984699] env[63593]: DEBUG nova.network.neutron [-] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1055.429214] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: d760b552-e2ff-46db-88eb-f5f58fc88239] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1055.487355] env[63593]: INFO nova.compute.manager [-] [instance: a223dd01-d07d-4ac5-acfc-2e03b4c7dff4] Took 1.02 seconds to deallocate network for instance. [ 1055.943544] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "d760b552-e2ff-46db-88eb-f5f58fc88239" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.235s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1056.447082] env[63593]: DEBUG nova.compute.manager [None req-04226f3f-5274-4c29-bbb8-eb0178d28bbd tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: 257c613c-8613-42e5-b400-b9c352a151f0] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1056.514936] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5e1a33cb-d2f9-4357-974c-053cfced6455 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "a223dd01-d07d-4ac5-acfc-2e03b4c7dff4" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.093s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1056.950551] env[63593]: DEBUG nova.compute.manager [None req-04226f3f-5274-4c29-bbb8-eb0178d28bbd tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] [instance: 257c613c-8613-42e5-b400-b9c352a151f0] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1057.462966] env[63593]: DEBUG oslo_concurrency.lockutils [None req-04226f3f-5274-4c29-bbb8-eb0178d28bbd tempest-MigrationsAdminTest-386021667 tempest-MigrationsAdminTest-386021667-project-member] Lock "257c613c-8613-42e5-b400-b9c352a151f0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 223.348s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1057.967465] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1058.490604] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1058.490861] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1058.492339] env[63593]: INFO nova.compute.claims [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1059.681859] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3717b6b-84f8-4496-aa71-c3ac3c19415a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1059.689960] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-816a5b9d-fbce-4777-8a68-5886cae8e5df {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1059.720961] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34f938de-0f62-46e1-aa0f-e25055482b7f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1059.728168] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73896263-85d8-4d9a-b691-1a3ed9a51e9e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1059.741167] env[63593]: DEBUG nova.compute.provider_tree [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1060.247045] env[63593]: DEBUG nova.scheduler.client.report [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1060.751784] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.261s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1060.752324] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1061.257352] env[63593]: DEBUG nova.compute.utils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1061.260071] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1061.260071] env[63593]: DEBUG nova.network.neutron [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1061.319327] env[63593]: DEBUG nova.policy [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c0a03b9520a349cb99065ce54ec21927', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '87f6487499be40a9b8bf821e1bd4de80', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1061.609297] env[63593]: DEBUG nova.network.neutron [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Successfully created port: c94090ea-79e6-42b3-b50f-d1afaef67396 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1061.762390] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1062.775018] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1062.801678] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1062.802376] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1062.802376] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1062.802507] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1062.802605] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1062.802755] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1062.803057] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1062.803180] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1062.803351] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1062.803523] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1062.803678] env[63593]: DEBUG nova.virt.hardware [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1062.804563] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dffe271-6aa3-4d23-9c65-ae9d80e9df3c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1062.812377] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d59d499-4712-484d-b39b-e8466b7461da {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1063.135015] env[63593]: DEBUG nova.compute.manager [req-e4c030c4-8fe2-4879-a1cb-ed05485d2f7d req-64f01974-d32d-495e-a50f-e2054133057b service nova] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Received event network-vif-plugged-c94090ea-79e6-42b3-b50f-d1afaef67396 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1063.135224] env[63593]: DEBUG oslo_concurrency.lockutils [req-e4c030c4-8fe2-4879-a1cb-ed05485d2f7d req-64f01974-d32d-495e-a50f-e2054133057b service nova] Acquiring lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1063.135441] env[63593]: DEBUG oslo_concurrency.lockutils [req-e4c030c4-8fe2-4879-a1cb-ed05485d2f7d req-64f01974-d32d-495e-a50f-e2054133057b service nova] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1063.135610] env[63593]: DEBUG oslo_concurrency.lockutils [req-e4c030c4-8fe2-4879-a1cb-ed05485d2f7d req-64f01974-d32d-495e-a50f-e2054133057b service nova] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1063.135777] env[63593]: DEBUG nova.compute.manager [req-e4c030c4-8fe2-4879-a1cb-ed05485d2f7d req-64f01974-d32d-495e-a50f-e2054133057b service nova] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] No waiting events found dispatching network-vif-plugged-c94090ea-79e6-42b3-b50f-d1afaef67396 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1063.135937] env[63593]: WARNING nova.compute.manager [req-e4c030c4-8fe2-4879-a1cb-ed05485d2f7d req-64f01974-d32d-495e-a50f-e2054133057b service nova] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Received unexpected event network-vif-plugged-c94090ea-79e6-42b3-b50f-d1afaef67396 for instance with vm_state building and task_state spawning. [ 1063.220375] env[63593]: DEBUG nova.network.neutron [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Successfully updated port: c94090ea-79e6-42b3-b50f-d1afaef67396 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1063.722718] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "refresh_cache-4e8f8346-70b0-4ae4-925c-50094ac5a07a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1063.722873] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "refresh_cache-4e8f8346-70b0-4ae4-925c-50094ac5a07a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1063.723045] env[63593]: DEBUG nova.network.neutron [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1064.260397] env[63593]: DEBUG nova.network.neutron [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1064.433795] env[63593]: DEBUG nova.network.neutron [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Updating instance_info_cache with network_info: [{"id": "c94090ea-79e6-42b3-b50f-d1afaef67396", "address": "fa:16:3e:28:e2:d4", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc94090ea-79", "ovs_interfaceid": "c94090ea-79e6-42b3-b50f-d1afaef67396", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1064.731871] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1064.936254] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "refresh_cache-4e8f8346-70b0-4ae4-925c-50094ac5a07a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1064.936652] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Instance network_info: |[{"id": "c94090ea-79e6-42b3-b50f-d1afaef67396", "address": "fa:16:3e:28:e2:d4", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc94090ea-79", "ovs_interfaceid": "c94090ea-79e6-42b3-b50f-d1afaef67396", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1064.937117] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:28:e2:d4', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a15de394-0367-4921-a5c1-6ac8615e3283', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'c94090ea-79e6-42b3-b50f-d1afaef67396', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1064.944800] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating folder: Project (87f6487499be40a9b8bf821e1bd4de80). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1064.945138] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-bafec4be-c70c-4fc9-b6da-be918fde58e3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1064.955994] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created folder: Project (87f6487499be40a9b8bf821e1bd4de80) in parent group-v291016. [ 1064.956199] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating folder: Instances. Parent ref: group-v291064. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1064.956426] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-e76c3b64-d4eb-418f-a978-4bed45c44f3b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1064.966512] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created folder: Instances in parent group-v291064. [ 1064.966857] env[63593]: DEBUG oslo.service.loopingcall [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1064.966943] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1064.967131] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-771c5abc-4000-441a-a751-3775b10ecddd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1064.985664] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1064.985664] env[63593]: value = "task-1368008" [ 1064.985664] env[63593]: _type = "Task" [ 1064.985664] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1064.992907] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368008, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1065.191580] env[63593]: DEBUG nova.compute.manager [req-8ac05c2d-ef6e-4c98-83b2-307b50d91a1e req-dc6eb728-22f3-411f-bfaf-901ea739cac6 service nova] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Received event network-changed-c94090ea-79e6-42b3-b50f-d1afaef67396 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1065.191703] env[63593]: DEBUG nova.compute.manager [req-8ac05c2d-ef6e-4c98-83b2-307b50d91a1e req-dc6eb728-22f3-411f-bfaf-901ea739cac6 service nova] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Refreshing instance network info cache due to event network-changed-c94090ea-79e6-42b3-b50f-d1afaef67396. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1065.191949] env[63593]: DEBUG oslo_concurrency.lockutils [req-8ac05c2d-ef6e-4c98-83b2-307b50d91a1e req-dc6eb728-22f3-411f-bfaf-901ea739cac6 service nova] Acquiring lock "refresh_cache-4e8f8346-70b0-4ae4-925c-50094ac5a07a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1065.192127] env[63593]: DEBUG oslo_concurrency.lockutils [req-8ac05c2d-ef6e-4c98-83b2-307b50d91a1e req-dc6eb728-22f3-411f-bfaf-901ea739cac6 service nova] Acquired lock "refresh_cache-4e8f8346-70b0-4ae4-925c-50094ac5a07a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1065.192285] env[63593]: DEBUG nova.network.neutron [req-8ac05c2d-ef6e-4c98-83b2-307b50d91a1e req-dc6eb728-22f3-411f-bfaf-901ea739cac6 service nova] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Refreshing network info cache for port c94090ea-79e6-42b3-b50f-d1afaef67396 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1065.495339] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368008, 'name': CreateVM_Task, 'duration_secs': 0.281843} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1065.495622] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1065.496182] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1065.496346] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1065.496662] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1065.496903] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-0ec0409f-be69-4d76-beb4-ca22d901518b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1065.501326] env[63593]: DEBUG oslo_vmware.api [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 1065.501326] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5272f37a-c287-10cf-502a-9dd52dab6685" [ 1065.501326] env[63593]: _type = "Task" [ 1065.501326] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1065.508296] env[63593]: DEBUG oslo_vmware.api [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5272f37a-c287-10cf-502a-9dd52dab6685, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1065.900871] env[63593]: DEBUG nova.network.neutron [req-8ac05c2d-ef6e-4c98-83b2-307b50d91a1e req-dc6eb728-22f3-411f-bfaf-901ea739cac6 service nova] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Updated VIF entry in instance network info cache for port c94090ea-79e6-42b3-b50f-d1afaef67396. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1065.901244] env[63593]: DEBUG nova.network.neutron [req-8ac05c2d-ef6e-4c98-83b2-307b50d91a1e req-dc6eb728-22f3-411f-bfaf-901ea739cac6 service nova] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Updating instance_info_cache with network_info: [{"id": "c94090ea-79e6-42b3-b50f-d1afaef67396", "address": "fa:16:3e:28:e2:d4", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc94090ea-79", "ovs_interfaceid": "c94090ea-79e6-42b3-b50f-d1afaef67396", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1066.011528] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1066.011660] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1066.011861] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1066.404175] env[63593]: DEBUG oslo_concurrency.lockutils [req-8ac05c2d-ef6e-4c98-83b2-307b50d91a1e req-dc6eb728-22f3-411f-bfaf-901ea739cac6 service nova] Releasing lock "refresh_cache-4e8f8346-70b0-4ae4-925c-50094ac5a07a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1075.857473] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquiring lock "41e38c8a-3aa1-4524-8820-27ba85187fe8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1075.858591] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1076.088198] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "442bcb9d-533f-4072-b967-791e3bc99643" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1076.088427] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "442bcb9d-533f-4072-b967-791e3bc99643" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1081.568944] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9aaaa7d9-378b-46a7-8590-1cf9d49bfb11 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Acquiring lock "1f3118b4-f510-48d3-88ba-230564a4e218" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1081.569285] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9aaaa7d9-378b-46a7-8590-1cf9d49bfb11 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "1f3118b4-f510-48d3-88ba-230564a4e218" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1087.204705] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e370cb6c-3042-4545-adf6-0f1d039085f5 tempest-ImagesNegativeTestJSON-1494673914 tempest-ImagesNegativeTestJSON-1494673914-project-member] Acquiring lock "2a07ea77-f690-4c34-96b2-8ff9ce603326" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1087.205041] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e370cb6c-3042-4545-adf6-0f1d039085f5 tempest-ImagesNegativeTestJSON-1494673914 tempest-ImagesNegativeTestJSON-1494673914-project-member] Lock "2a07ea77-f690-4c34-96b2-8ff9ce603326" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1095.396964] env[63593]: WARNING oslo_vmware.rw_handles [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1095.396964] env[63593]: ERROR oslo_vmware.rw_handles [ 1095.397636] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/52919336-80f9-4236-85a0-e4bb60e405ee/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1095.399578] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1095.399812] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Copying Virtual Disk [datastore2] vmware_temp/52919336-80f9-4236-85a0-e4bb60e405ee/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/52919336-80f9-4236-85a0-e4bb60e405ee/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1095.400120] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-5d906ab9-8af0-49d7-b097-9497470412f8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1095.410085] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for the task: (returnval){ [ 1095.410085] env[63593]: value = "task-1368009" [ 1095.410085] env[63593]: _type = "Task" [ 1095.410085] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1095.417840] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': task-1368009, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1095.920141] env[63593]: DEBUG oslo_vmware.exceptions [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1095.920449] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1095.921053] env[63593]: ERROR nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1095.921053] env[63593]: Faults: ['InvalidArgument'] [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Traceback (most recent call last): [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] yield resources [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] self.driver.spawn(context, instance, image_meta, [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] self._fetch_image_if_missing(context, vi) [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] image_cache(vi, tmp_image_ds_loc) [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] vm_util.copy_virtual_disk( [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] session._wait_for_task(vmdk_copy_task) [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] return self.wait_for_task(task_ref) [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] return evt.wait() [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] result = hub.switch() [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] return self.greenlet.switch() [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] self.f(*self.args, **self.kw) [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] raise exceptions.translate_fault(task_info.error) [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Faults: ['InvalidArgument'] [ 1095.921053] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] [ 1095.922232] env[63593]: INFO nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Terminating instance [ 1095.923039] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1095.923256] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1095.923493] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e6fafd63-43fb-4e8c-b387-eea5f472af2c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1095.925661] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1095.925859] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1095.926617] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cdf0da79-b234-49b8-aaa9-d85949ebf893 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1095.934134] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1095.935032] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0cd87a63-f4d7-4e86-9c95-ca6e7810d39f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1095.936839] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1095.937019] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1095.937946] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c231dd7c-7fe6-436d-8c04-9a17354f98b4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1095.942985] env[63593]: DEBUG oslo_vmware.api [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Waiting for the task: (returnval){ [ 1095.942985] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5202fee5-0e7d-c3df-3947-c8f4dbd92511" [ 1095.942985] env[63593]: _type = "Task" [ 1095.942985] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1095.954162] env[63593]: DEBUG oslo_vmware.api [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5202fee5-0e7d-c3df-3947-c8f4dbd92511, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1096.006275] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1096.006584] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1096.006852] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Deleting the datastore file [datastore2] d3a4902f-1829-427c-9f37-6e83ae7265e3 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1096.007114] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-eed07d0b-8bea-44c0-93ab-2906ebb0c7b6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1096.013902] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for the task: (returnval){ [ 1096.013902] env[63593]: value = "task-1368011" [ 1096.013902] env[63593]: _type = "Task" [ 1096.013902] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1096.023167] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': task-1368011, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1096.453023] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1096.453308] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Creating directory with path [datastore2] vmware_temp/40b3b70d-e1aa-485e-bf6d-bdee18081f7e/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1096.453532] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-cbaf75ac-3857-4964-bc0f-fa7eddd64d23 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1096.464281] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Created directory with path [datastore2] vmware_temp/40b3b70d-e1aa-485e-bf6d-bdee18081f7e/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1096.464460] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Fetch image to [datastore2] vmware_temp/40b3b70d-e1aa-485e-bf6d-bdee18081f7e/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1096.464625] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/40b3b70d-e1aa-485e-bf6d-bdee18081f7e/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1096.465311] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8490c8eb-b861-46cf-8bf8-4ff7f1f80b6d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1096.471334] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8aca8f51-b491-4d07-a0a6-71dab1aeb91b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1096.479914] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ddd2cb16-28c0-4ac6-8e71-c6acc8dbec07 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1096.509844] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83491732-9a05-4099-a58c-b6c8a3e20531 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1096.517714] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-40e20e16-e067-4364-b4cf-c616d1f84506 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1096.523620] env[63593]: DEBUG oslo_vmware.api [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Task: {'id': task-1368011, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.063295} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1096.523848] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1096.524058] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1096.524233] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1096.524401] env[63593]: INFO nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1096.526432] env[63593]: DEBUG nova.compute.claims [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1096.526601] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1096.526808] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1096.539788] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1096.589509] env[63593]: DEBUG oslo_vmware.rw_handles [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/40b3b70d-e1aa-485e-bf6d-bdee18081f7e/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1096.650805] env[63593]: DEBUG oslo_vmware.rw_handles [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1096.651076] env[63593]: DEBUG oslo_vmware.rw_handles [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/40b3b70d-e1aa-485e-bf6d-bdee18081f7e/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1097.218554] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65ca8f3a-0dab-4224-9797-1606f37145c6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1097.227463] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25b8264b-90d9-4b4a-9c2b-b6b3ecd7d5f0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1097.259601] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29c6bee1-d88e-490f-839e-0374823f799d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1097.266902] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d4cace78-99fd-4483-84a8-14bff1bbe25e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1097.280154] env[63593]: DEBUG nova.compute.provider_tree [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1097.783496] env[63593]: DEBUG nova.scheduler.client.report [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1098.288931] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.762s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1098.289580] env[63593]: ERROR nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1098.289580] env[63593]: Faults: ['InvalidArgument'] [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Traceback (most recent call last): [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] self.driver.spawn(context, instance, image_meta, [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] self._fetch_image_if_missing(context, vi) [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] image_cache(vi, tmp_image_ds_loc) [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] vm_util.copy_virtual_disk( [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] session._wait_for_task(vmdk_copy_task) [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] return self.wait_for_task(task_ref) [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] return evt.wait() [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] result = hub.switch() [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] return self.greenlet.switch() [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] self.f(*self.args, **self.kw) [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] raise exceptions.translate_fault(task_info.error) [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Faults: ['InvalidArgument'] [ 1098.289580] env[63593]: ERROR nova.compute.manager [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] [ 1098.290908] env[63593]: DEBUG nova.compute.utils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1098.292206] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Build of instance d3a4902f-1829-427c-9f37-6e83ae7265e3 was re-scheduled: A specified parameter was not correct: fileType [ 1098.292206] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1098.292604] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1098.292752] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1098.292929] env[63593]: DEBUG nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1098.293107] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1099.033608] env[63593]: DEBUG nova.network.neutron [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1099.536258] env[63593]: INFO nova.compute.manager [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Took 1.24 seconds to deallocate network for instance. [ 1100.572409] env[63593]: INFO nova.scheduler.client.report [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Deleted allocations for instance d3a4902f-1829-427c-9f37-6e83ae7265e3 [ 1101.080712] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2934856b-9df3-4861-b5f3-785380f82cb1 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "d3a4902f-1829-427c-9f37-6e83ae7265e3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 309.292s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1101.081699] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "d3a4902f-1829-427c-9f37-6e83ae7265e3" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 112.438s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1101.081932] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Acquiring lock "d3a4902f-1829-427c-9f37-6e83ae7265e3-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1101.082156] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "d3a4902f-1829-427c-9f37-6e83ae7265e3-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1101.082327] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "d3a4902f-1829-427c-9f37-6e83ae7265e3-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1101.086847] env[63593]: INFO nova.compute.manager [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Terminating instance [ 1101.088628] env[63593]: DEBUG nova.compute.manager [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1101.088825] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1101.089098] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-106d1cb1-b687-4afa-a919-2748792d1bf0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1101.099300] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae15b2d3-ebf2-415e-8704-b8b6b6b86604 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1101.125513] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d3a4902f-1829-427c-9f37-6e83ae7265e3 could not be found. [ 1101.125721] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1101.125897] env[63593]: INFO nova.compute.manager [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1101.126148] env[63593]: DEBUG oslo.service.loopingcall [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1101.126375] env[63593]: DEBUG nova.compute.manager [-] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1101.126466] env[63593]: DEBUG nova.network.neutron [-] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1101.585270] env[63593]: DEBUG nova.compute.manager [None req-d05df664-cf9b-4dd9-90de-1fa791a5346c tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 0c9d6b46-e1f1-4dcc-b5bf-c6ca520d4e0f] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1101.646269] env[63593]: DEBUG nova.network.neutron [-] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1102.090606] env[63593]: DEBUG nova.compute.manager [None req-d05df664-cf9b-4dd9-90de-1fa791a5346c tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 0c9d6b46-e1f1-4dcc-b5bf-c6ca520d4e0f] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1102.152537] env[63593]: INFO nova.compute.manager [-] [instance: d3a4902f-1829-427c-9f37-6e83ae7265e3] Took 1.03 seconds to deallocate network for instance. [ 1102.613866] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d05df664-cf9b-4dd9-90de-1fa791a5346c tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "0c9d6b46-e1f1-4dcc-b5bf-c6ca520d4e0f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.132s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1103.114565] env[63593]: DEBUG nova.compute.manager [None req-48c5bd4b-ae92-4455-ad4e-0909267f969d tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 4dd65fb0-e079-4520-8809-dfd275d876f7] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1103.181370] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e0d9745e-49ce-491e-bc0f-895e20ef8a40 tempest-MultipleCreateTestJSON-424064613 tempest-MultipleCreateTestJSON-424064613-project-member] Lock "d3a4902f-1829-427c-9f37-6e83ae7265e3" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.100s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1103.620918] env[63593]: DEBUG nova.compute.manager [None req-48c5bd4b-ae92-4455-ad4e-0909267f969d tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 4dd65fb0-e079-4520-8809-dfd275d876f7] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1104.135241] env[63593]: DEBUG oslo_concurrency.lockutils [None req-48c5bd4b-ae92-4455-ad4e-0909267f969d tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "4dd65fb0-e079-4520-8809-dfd275d876f7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.748s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1104.638909] env[63593]: DEBUG nova.compute.manager [None req-96b8f73d-4a6f-4d88-8937-917863b694fb tempest-ServerMetadataNegativeTestJSON-2066118354 tempest-ServerMetadataNegativeTestJSON-2066118354-project-member] [instance: a4c51198-1e5d-4a59-8a9b-a997872e372b] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1105.145862] env[63593]: DEBUG nova.compute.manager [None req-96b8f73d-4a6f-4d88-8937-917863b694fb tempest-ServerMetadataNegativeTestJSON-2066118354 tempest-ServerMetadataNegativeTestJSON-2066118354-project-member] [instance: a4c51198-1e5d-4a59-8a9b-a997872e372b] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1105.659247] env[63593]: DEBUG oslo_concurrency.lockutils [None req-96b8f73d-4a6f-4d88-8937-917863b694fb tempest-ServerMetadataNegativeTestJSON-2066118354 tempest-ServerMetadataNegativeTestJSON-2066118354-project-member] Lock "a4c51198-1e5d-4a59-8a9b-a997872e372b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.009s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1106.161554] env[63593]: DEBUG nova.compute.manager [None req-a96bc521-cec1-413b-84df-d6323f356c82 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 0cdcc9d3-9963-49b5-8744-241b865335e2] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1106.665467] env[63593]: DEBUG nova.compute.manager [None req-a96bc521-cec1-413b-84df-d6323f356c82 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 0cdcc9d3-9963-49b5-8744-241b865335e2] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1107.178437] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a96bc521-cec1-413b-84df-d6323f356c82 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "0cdcc9d3-9963-49b5-8744-241b865335e2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 222.639s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1107.681656] env[63593]: DEBUG nova.compute.manager [None req-3bd3c13f-ce31-468b-b893-2e083f8084ba tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 38e38231-7b05-43c7-8de2-993a03ed4f57] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1108.186470] env[63593]: DEBUG nova.compute.manager [None req-3bd3c13f-ce31-468b-b893-2e083f8084ba tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] [instance: 38e38231-7b05-43c7-8de2-993a03ed4f57] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1108.699200] env[63593]: DEBUG oslo_concurrency.lockutils [None req-3bd3c13f-ce31-468b-b893-2e083f8084ba tempest-ImagesTestJSON-815993396 tempest-ImagesTestJSON-815993396-project-member] Lock "38e38231-7b05-43c7-8de2-993a03ed4f57" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 222.872s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1108.802110] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1108.802355] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1108.802502] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1108.802620] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1109.202101] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1109.307317] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1109.307484] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1109.307614] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1109.307739] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1109.307861] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b053676a-81b8-4345-b257-1d67976fd077] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1109.307982] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1109.308160] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1109.308313] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1109.308461] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1109.308583] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1109.309165] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.309395] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.309554] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.309702] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.309843] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.309986] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.310131] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1109.310283] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1109.725807] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1109.725807] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1109.727042] env[63593]: INFO nova.compute.claims [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1109.813042] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1110.890266] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5182c505-052b-4aa1-93c2-808480aed087 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1110.898063] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d941174d-4e66-46d8-a487-d93e627126e5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1110.926854] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90048b49-25ca-47ad-a85a-9eca0ebe752d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1110.933613] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-915da250-fa83-4bbc-93b4-af08daa3c774 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1110.946327] env[63593]: DEBUG nova.compute.provider_tree [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1111.450071] env[63593]: DEBUG nova.scheduler.client.report [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1111.954831] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.229s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1111.955374] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1111.957867] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 2.145s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1111.957981] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1111.958148] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1111.959206] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7dd4fefb-1b6f-4dec-a07b-b54081a45075 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1111.967361] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-026a11fd-4ecd-4130-ae96-832d648841de {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1111.982324] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df191f86-2077-4a83-a779-1c8877770ec5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1111.988819] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84c0417e-c54c-4b4e-ba17-b028dac146c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1112.017508] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181473MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1112.017681] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1112.017830] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1112.462209] env[63593]: DEBUG nova.compute.utils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1112.463235] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1112.463463] env[63593]: DEBUG nova.network.neutron [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1112.505961] env[63593]: DEBUG nova.policy [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f140a9faaeab429bab37595415f0ec34', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3fa7a976fe3c4bb5a95b0732ca7d62f6', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1112.824273] env[63593]: DEBUG nova.network.neutron [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Successfully created port: 0d31653a-9c19-4954-bcd6-156d570ca93e {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1112.966279] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1113.049143] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.049306] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fbd620a6-3743-4737-98b7-b13928b6587d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.049426] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.049541] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 328b91c6-4ae4-442f-ac87-519658a812ff actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.049656] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.049865] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.049865] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.049977] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.050103] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.050214] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1113.553270] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1113.976627] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1114.000623] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1114.000854] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1114.001399] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1114.001399] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1114.001524] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1114.001556] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1114.001753] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1114.001905] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1114.002080] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1114.002422] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1114.002628] env[63593]: DEBUG nova.virt.hardware [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1114.003537] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d87ab86-9e3b-4f58-9367-a3607fddf385 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1114.012109] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c86d4df7-3692-43f9-a5ca-d7c644a55156 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1114.059807] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1114.205731] env[63593]: DEBUG nova.compute.manager [req-78bcf18e-2a9e-4c05-8098-5f0cc72ccf2e req-e26795af-852c-4ccd-9cc8-2c94a1d7ec7d service nova] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Received event network-vif-plugged-0d31653a-9c19-4954-bcd6-156d570ca93e {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1114.206670] env[63593]: DEBUG oslo_concurrency.lockutils [req-78bcf18e-2a9e-4c05-8098-5f0cc72ccf2e req-e26795af-852c-4ccd-9cc8-2c94a1d7ec7d service nova] Acquiring lock "488c4bca-8279-4de7-8259-238497cbd011-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1114.206670] env[63593]: DEBUG oslo_concurrency.lockutils [req-78bcf18e-2a9e-4c05-8098-5f0cc72ccf2e req-e26795af-852c-4ccd-9cc8-2c94a1d7ec7d service nova] Lock "488c4bca-8279-4de7-8259-238497cbd011-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1114.206670] env[63593]: DEBUG oslo_concurrency.lockutils [req-78bcf18e-2a9e-4c05-8098-5f0cc72ccf2e req-e26795af-852c-4ccd-9cc8-2c94a1d7ec7d service nova] Lock "488c4bca-8279-4de7-8259-238497cbd011-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1114.206670] env[63593]: DEBUG nova.compute.manager [req-78bcf18e-2a9e-4c05-8098-5f0cc72ccf2e req-e26795af-852c-4ccd-9cc8-2c94a1d7ec7d service nova] [instance: 488c4bca-8279-4de7-8259-238497cbd011] No waiting events found dispatching network-vif-plugged-0d31653a-9c19-4954-bcd6-156d570ca93e {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1114.206670] env[63593]: WARNING nova.compute.manager [req-78bcf18e-2a9e-4c05-8098-5f0cc72ccf2e req-e26795af-852c-4ccd-9cc8-2c94a1d7ec7d service nova] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Received unexpected event network-vif-plugged-0d31653a-9c19-4954-bcd6-156d570ca93e for instance with vm_state building and task_state spawning. [ 1114.288778] env[63593]: DEBUG nova.network.neutron [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Successfully updated port: 0d31653a-9c19-4954-bcd6-156d570ca93e {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1114.562780] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1114.794297] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquiring lock "refresh_cache-488c4bca-8279-4de7-8259-238497cbd011" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1114.794297] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquired lock "refresh_cache-488c4bca-8279-4de7-8259-238497cbd011" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1114.794453] env[63593]: DEBUG nova.network.neutron [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1115.065452] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 1f3118b4-f510-48d3-88ba-230564a4e218 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1115.324582] env[63593]: DEBUG nova.network.neutron [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1115.446120] env[63593]: DEBUG nova.network.neutron [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Updating instance_info_cache with network_info: [{"id": "0d31653a-9c19-4954-bcd6-156d570ca93e", "address": "fa:16:3e:3e:e1:f5", "network": {"id": "f7e27f2c-ef93-4b48-a4dc-3ddf5819bc05", "bridge": "br-int", "label": "tempest-ServersNegativeTestMultiTenantJSON-347975007-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3fa7a976fe3c4bb5a95b0732ca7d62f6", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "6a31b224-77d7-48a4-af87-312758611b8e", "external-id": "nsx-vlan-transportzone-761", "segmentation_id": 761, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap0d31653a-9c", "ovs_interfaceid": "0d31653a-9c19-4954-bcd6-156d570ca93e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1115.568785] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2a07ea77-f690-4c34-96b2-8ff9ce603326 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1115.569060] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1115.569226] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1115.742923] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac947d60-bd17-4e8e-86c2-b4bb0776da71 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1115.747454] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d38fb131-51fb-4260-badf-3882b1d6031e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1115.778511] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d27709f-cb4c-466e-9ead-a6b76dfcd5a7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1115.785845] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a9ed91a-b4b4-4c76-be19-e2fb625c34e0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1115.798684] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1115.948509] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Releasing lock "refresh_cache-488c4bca-8279-4de7-8259-238497cbd011" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1115.948821] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Instance network_info: |[{"id": "0d31653a-9c19-4954-bcd6-156d570ca93e", "address": "fa:16:3e:3e:e1:f5", "network": {"id": "f7e27f2c-ef93-4b48-a4dc-3ddf5819bc05", "bridge": "br-int", "label": "tempest-ServersNegativeTestMultiTenantJSON-347975007-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3fa7a976fe3c4bb5a95b0732ca7d62f6", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "6a31b224-77d7-48a4-af87-312758611b8e", "external-id": "nsx-vlan-transportzone-761", "segmentation_id": 761, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap0d31653a-9c", "ovs_interfaceid": "0d31653a-9c19-4954-bcd6-156d570ca93e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1115.949875] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:3e:e1:f5', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '6a31b224-77d7-48a4-af87-312758611b8e', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '0d31653a-9c19-4954-bcd6-156d570ca93e', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1115.956498] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Creating folder: Project (3fa7a976fe3c4bb5a95b0732ca7d62f6). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1115.956769] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6164c5f1-fcfd-429c-b93b-6e28d6447647 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1115.967916] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Created folder: Project (3fa7a976fe3c4bb5a95b0732ca7d62f6) in parent group-v291016. [ 1115.968107] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Creating folder: Instances. Parent ref: group-v291067. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1115.968332] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-07d4d178-8073-4ecd-9348-d462af6bfb67 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1115.977183] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Created folder: Instances in parent group-v291067. [ 1115.977447] env[63593]: DEBUG oslo.service.loopingcall [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1115.977644] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1115.977831] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-b2403cb7-2564-447c-8c6e-316dde44d24a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1115.996010] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1115.996010] env[63593]: value = "task-1368014" [ 1115.996010] env[63593]: _type = "Task" [ 1115.996010] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1116.002822] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368014, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1116.234205] env[63593]: DEBUG nova.compute.manager [req-b5362e84-44e7-47d9-8455-ea220cfc5f3d req-4fafcfed-3c83-4a54-8815-46cf6103d586 service nova] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Received event network-changed-0d31653a-9c19-4954-bcd6-156d570ca93e {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1116.234505] env[63593]: DEBUG nova.compute.manager [req-b5362e84-44e7-47d9-8455-ea220cfc5f3d req-4fafcfed-3c83-4a54-8815-46cf6103d586 service nova] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Refreshing instance network info cache due to event network-changed-0d31653a-9c19-4954-bcd6-156d570ca93e. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1116.234625] env[63593]: DEBUG oslo_concurrency.lockutils [req-b5362e84-44e7-47d9-8455-ea220cfc5f3d req-4fafcfed-3c83-4a54-8815-46cf6103d586 service nova] Acquiring lock "refresh_cache-488c4bca-8279-4de7-8259-238497cbd011" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1116.234763] env[63593]: DEBUG oslo_concurrency.lockutils [req-b5362e84-44e7-47d9-8455-ea220cfc5f3d req-4fafcfed-3c83-4a54-8815-46cf6103d586 service nova] Acquired lock "refresh_cache-488c4bca-8279-4de7-8259-238497cbd011" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1116.234918] env[63593]: DEBUG nova.network.neutron [req-b5362e84-44e7-47d9-8455-ea220cfc5f3d req-4fafcfed-3c83-4a54-8815-46cf6103d586 service nova] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Refreshing network info cache for port 0d31653a-9c19-4954-bcd6-156d570ca93e {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1116.303044] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1116.506384] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368014, 'name': CreateVM_Task, 'duration_secs': 0.290127} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1116.506384] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1116.506966] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1116.507145] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1116.507458] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1116.507696] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5d549ca5-c1d3-4969-a85f-034665c52b28 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1116.511649] env[63593]: DEBUG oslo_vmware.api [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Waiting for the task: (returnval){ [ 1116.511649] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52330704-26e6-8c92-8079-e902c87fdbc5" [ 1116.511649] env[63593]: _type = "Task" [ 1116.511649] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1116.518604] env[63593]: DEBUG oslo_vmware.api [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52330704-26e6-8c92-8079-e902c87fdbc5, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1116.807222] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1116.807441] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 4.790s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1117.025067] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1117.025067] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1117.025067] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1117.153798] env[63593]: DEBUG nova.network.neutron [req-b5362e84-44e7-47d9-8455-ea220cfc5f3d req-4fafcfed-3c83-4a54-8815-46cf6103d586 service nova] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Updated VIF entry in instance network info cache for port 0d31653a-9c19-4954-bcd6-156d570ca93e. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1117.154698] env[63593]: DEBUG nova.network.neutron [req-b5362e84-44e7-47d9-8455-ea220cfc5f3d req-4fafcfed-3c83-4a54-8815-46cf6103d586 service nova] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Updating instance_info_cache with network_info: [{"id": "0d31653a-9c19-4954-bcd6-156d570ca93e", "address": "fa:16:3e:3e:e1:f5", "network": {"id": "f7e27f2c-ef93-4b48-a4dc-3ddf5819bc05", "bridge": "br-int", "label": "tempest-ServersNegativeTestMultiTenantJSON-347975007-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3fa7a976fe3c4bb5a95b0732ca7d62f6", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "6a31b224-77d7-48a4-af87-312758611b8e", "external-id": "nsx-vlan-transportzone-761", "segmentation_id": 761, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap0d31653a-9c", "ovs_interfaceid": "0d31653a-9c19-4954-bcd6-156d570ca93e", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1117.657630] env[63593]: DEBUG oslo_concurrency.lockutils [req-b5362e84-44e7-47d9-8455-ea220cfc5f3d req-4fafcfed-3c83-4a54-8815-46cf6103d586 service nova] Releasing lock "refresh_cache-488c4bca-8279-4de7-8259-238497cbd011" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1143.612373] env[63593]: WARNING oslo_vmware.rw_handles [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1143.612373] env[63593]: ERROR oslo_vmware.rw_handles [ 1143.613034] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/40b3b70d-e1aa-485e-bf6d-bdee18081f7e/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1143.615096] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1143.615349] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Copying Virtual Disk [datastore2] vmware_temp/40b3b70d-e1aa-485e-bf6d-bdee18081f7e/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/40b3b70d-e1aa-485e-bf6d-bdee18081f7e/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1143.615629] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-4ea4777a-c5df-4cac-ac1a-4abeae8a16af {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1143.626363] env[63593]: DEBUG oslo_vmware.api [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Waiting for the task: (returnval){ [ 1143.626363] env[63593]: value = "task-1368015" [ 1143.626363] env[63593]: _type = "Task" [ 1143.626363] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1143.633907] env[63593]: DEBUG oslo_vmware.api [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Task: {'id': task-1368015, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1144.135905] env[63593]: DEBUG oslo_vmware.exceptions [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1144.136270] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1144.136805] env[63593]: ERROR nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1144.136805] env[63593]: Faults: ['InvalidArgument'] [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Traceback (most recent call last): [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] yield resources [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] self.driver.spawn(context, instance, image_meta, [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] self._fetch_image_if_missing(context, vi) [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] image_cache(vi, tmp_image_ds_loc) [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] vm_util.copy_virtual_disk( [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] session._wait_for_task(vmdk_copy_task) [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] return self.wait_for_task(task_ref) [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] return evt.wait() [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] result = hub.switch() [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] return self.greenlet.switch() [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] self.f(*self.args, **self.kw) [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] raise exceptions.translate_fault(task_info.error) [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Faults: ['InvalidArgument'] [ 1144.136805] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] [ 1144.137998] env[63593]: INFO nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Terminating instance [ 1144.139025] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1144.139025] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1144.139179] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-69a14251-7a8b-4fa1-9182-9b7bfe0f3d71 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.141476] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1144.141672] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1144.142407] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa972504-dc5e-44c3-a82e-351f06cfa080 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.148809] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1144.149031] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-fa4a99fe-90ad-46ad-84b6-99947c760f4a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.151085] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1144.151256] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1144.152226] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c3367d9d-6ef0-4f3e-bb66-15c8dbea263f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.156870] env[63593]: DEBUG oslo_vmware.api [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Waiting for the task: (returnval){ [ 1144.156870] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5210faf8-39e2-4d6e-247e-b502fb35063a" [ 1144.156870] env[63593]: _type = "Task" [ 1144.156870] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1144.167357] env[63593]: DEBUG oslo_vmware.api [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5210faf8-39e2-4d6e-247e-b502fb35063a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1144.227182] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1144.227403] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1144.227582] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Deleting the datastore file [datastore2] efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1144.227852] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-7bd6229a-f5ed-411e-9ed3-68d97c553025 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.235035] env[63593]: DEBUG oslo_vmware.api [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Waiting for the task: (returnval){ [ 1144.235035] env[63593]: value = "task-1368017" [ 1144.235035] env[63593]: _type = "Task" [ 1144.235035] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1144.242637] env[63593]: DEBUG oslo_vmware.api [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Task: {'id': task-1368017, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1144.667251] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1144.667563] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Creating directory with path [datastore2] vmware_temp/367d0437-ac08-405b-8669-99d0f4cb480c/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1144.670064] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-3baec46b-f4ac-4c7d-9654-b93b97447784 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.678849] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Created directory with path [datastore2] vmware_temp/367d0437-ac08-405b-8669-99d0f4cb480c/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1144.679131] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Fetch image to [datastore2] vmware_temp/367d0437-ac08-405b-8669-99d0f4cb480c/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1144.679366] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/367d0437-ac08-405b-8669-99d0f4cb480c/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1144.680410] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba1c8f6e-87e1-440e-ae17-0964e1acb500 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.689747] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-006ee1fa-32c1-4432-8924-6ecc6cd63b65 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.702105] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f98ccbd5-5547-442a-868a-821062417200 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.748041] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5537efa3-7f26-45e9-9fa0-b8eac0e26341 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.755170] env[63593]: DEBUG oslo_vmware.api [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Task: {'id': task-1368017, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.076782} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1144.756634] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1144.756825] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1144.756992] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1144.757178] env[63593]: INFO nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1144.758937] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-bb9023d0-4456-44a6-a538-0d7c7f93b5c6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.760762] env[63593]: DEBUG nova.compute.claims [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1144.760935] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1144.761158] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1144.784099] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1144.837975] env[63593]: DEBUG oslo_vmware.rw_handles [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/367d0437-ac08-405b-8669-99d0f4cb480c/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1144.898570] env[63593]: DEBUG oslo_vmware.rw_handles [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1144.899969] env[63593]: DEBUG oslo_vmware.rw_handles [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/367d0437-ac08-405b-8669-99d0f4cb480c/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1145.432112] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d1544cf-afbe-406f-b00d-4216d2bb134e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1145.439786] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f41c32a4-0731-415b-8328-01fadde44c17 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1145.469128] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d55920a-c030-4dd4-8615-21bf3baff3c9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1145.476069] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7dc5056f-c8ee-4679-9615-59d4292f6b61 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1145.488403] env[63593]: DEBUG nova.compute.provider_tree [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1145.993219] env[63593]: DEBUG nova.scheduler.client.report [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1146.497234] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.736s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1146.497836] env[63593]: ERROR nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1146.497836] env[63593]: Faults: ['InvalidArgument'] [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Traceback (most recent call last): [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] self.driver.spawn(context, instance, image_meta, [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] self._fetch_image_if_missing(context, vi) [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] image_cache(vi, tmp_image_ds_loc) [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] vm_util.copy_virtual_disk( [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] session._wait_for_task(vmdk_copy_task) [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] return self.wait_for_task(task_ref) [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] return evt.wait() [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] result = hub.switch() [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] return self.greenlet.switch() [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] self.f(*self.args, **self.kw) [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] raise exceptions.translate_fault(task_info.error) [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Faults: ['InvalidArgument'] [ 1146.497836] env[63593]: ERROR nova.compute.manager [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] [ 1146.498829] env[63593]: DEBUG nova.compute.utils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1146.500376] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Build of instance efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 was re-scheduled: A specified parameter was not correct: fileType [ 1146.500376] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1146.500768] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1146.500938] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1146.501119] env[63593]: DEBUG nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1146.501281] env[63593]: DEBUG nova.network.neutron [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1147.243396] env[63593]: DEBUG nova.network.neutron [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1147.747383] env[63593]: INFO nova.compute.manager [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Took 1.25 seconds to deallocate network for instance. [ 1148.781137] env[63593]: INFO nova.scheduler.client.report [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Deleted allocations for instance efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 [ 1149.289918] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f5ac8e2e-9761-457c-b00f-1876dedcd2e2 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 356.711s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1149.291713] env[63593]: DEBUG oslo_concurrency.lockutils [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 161.378s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1149.292043] env[63593]: DEBUG oslo_concurrency.lockutils [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Acquiring lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1149.292356] env[63593]: DEBUG oslo_concurrency.lockutils [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1149.292602] env[63593]: DEBUG oslo_concurrency.lockutils [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1149.296082] env[63593]: INFO nova.compute.manager [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Terminating instance [ 1149.297866] env[63593]: DEBUG nova.compute.manager [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1149.298170] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1149.298645] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a678ac79-26a8-49dc-afe0-ccac976ffee3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1149.309273] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8efd327f-e327-4fb6-be9c-c9b80909462c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1149.335774] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1 could not be found. [ 1149.335977] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1149.336171] env[63593]: INFO nova.compute.manager [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1149.336472] env[63593]: DEBUG oslo.service.loopingcall [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1149.336649] env[63593]: DEBUG nova.compute.manager [-] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1149.336748] env[63593]: DEBUG nova.network.neutron [-] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1149.796282] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1149.856317] env[63593]: DEBUG nova.network.neutron [-] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1150.012388] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1150.012621] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1150.012849] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1150.012983] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances with incomplete migration {{(pid=63593) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11264}} [ 1150.320584] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1150.320833] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1150.322330] env[63593]: INFO nova.compute.claims [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1150.359038] env[63593]: INFO nova.compute.manager [-] [instance: efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1] Took 1.02 seconds to deallocate network for instance. [ 1151.015810] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1151.016292] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1151.016478] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1151.390648] env[63593]: DEBUG oslo_concurrency.lockutils [None req-68cb9a51-5413-44cd-b570-ff1f287419a8 tempest-ServerAddressesNegativeTestJSON-1882808513 tempest-ServerAddressesNegativeTestJSON-1882808513-project-member] Lock "efc3ecd8-6bd6-4ef1-a7b3-3d9d25e9a5f1" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.099s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1151.520815] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.520979] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.521134] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.521260] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b053676a-81b8-4345-b257-1d67976fd077] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.521453] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.521551] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.521690] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.521827] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.521965] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.522112] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1151.522270] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1151.525035] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1151.537864] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f22908ce-2152-4864-974a-e8e99b7dad5c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1151.549435] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37c5ed77-0f3c-4646-a767-e19010507e35 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1151.580886] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77ec5bc6-a58a-4ea7-a9bf-c8b3823feae1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1151.588601] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-befdd448-0661-4e02-9b6a-c39c7e724277 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1151.603474] env[63593]: DEBUG nova.compute.provider_tree [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1152.106480] env[63593]: DEBUG nova.scheduler.client.report [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1152.485216] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "34867c6c-b8ae-401b-b6f4-ce36bea956ae" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1152.485442] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "34867c6c-b8ae-401b-b6f4-ce36bea956ae" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1152.514569] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1152.514682] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1152.612561] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.292s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1152.613127] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1153.013050] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1153.013295] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1153.117909] env[63593]: DEBUG nova.compute.utils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1153.119299] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1153.119553] env[63593]: DEBUG nova.network.neutron [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1153.156840] env[63593]: DEBUG nova.policy [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '657b14e9d117498485d033419916b1e9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6c739264d695443f935d41b24e6b484e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1153.390246] env[63593]: DEBUG nova.network.neutron [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Successfully created port: bcbc734c-3414-4396-9bc7-a344f91cb894 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1153.622741] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1154.008485] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1154.012183] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1154.012397] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1154.012645] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11226}} [ 1154.515143] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] There are 0 instances to clean {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11235}} [ 1154.632274] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1154.656552] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1154.656747] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1154.656890] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1154.657075] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1154.657226] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1154.657368] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1154.657572] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1154.657727] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1154.657889] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1154.658055] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1154.658226] env[63593]: DEBUG nova.virt.hardware [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1154.659089] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7a4943c-111d-45fc-97cc-2cbe7d56f441 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1154.667176] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-257cfad0-2a14-411a-b498-c5bb5a39b64e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1154.972920] env[63593]: DEBUG nova.compute.manager [req-19828a6d-396c-4fdb-808e-b294220c6b49 req-6a81c45f-5d9c-4353-ba5c-4bbaab59a71f service nova] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Received event network-vif-plugged-bcbc734c-3414-4396-9bc7-a344f91cb894 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1154.973160] env[63593]: DEBUG oslo_concurrency.lockutils [req-19828a6d-396c-4fdb-808e-b294220c6b49 req-6a81c45f-5d9c-4353-ba5c-4bbaab59a71f service nova] Acquiring lock "a55c8243-3413-435e-89cb-6a2bb65b5292-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1154.973362] env[63593]: DEBUG oslo_concurrency.lockutils [req-19828a6d-396c-4fdb-808e-b294220c6b49 req-6a81c45f-5d9c-4353-ba5c-4bbaab59a71f service nova] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1154.973936] env[63593]: DEBUG oslo_concurrency.lockutils [req-19828a6d-396c-4fdb-808e-b294220c6b49 req-6a81c45f-5d9c-4353-ba5c-4bbaab59a71f service nova] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1154.973936] env[63593]: DEBUG nova.compute.manager [req-19828a6d-396c-4fdb-808e-b294220c6b49 req-6a81c45f-5d9c-4353-ba5c-4bbaab59a71f service nova] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] No waiting events found dispatching network-vif-plugged-bcbc734c-3414-4396-9bc7-a344f91cb894 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1154.973936] env[63593]: WARNING nova.compute.manager [req-19828a6d-396c-4fdb-808e-b294220c6b49 req-6a81c45f-5d9c-4353-ba5c-4bbaab59a71f service nova] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Received unexpected event network-vif-plugged-bcbc734c-3414-4396-9bc7-a344f91cb894 for instance with vm_state building and task_state spawning. [ 1155.031341] env[63593]: DEBUG nova.network.neutron [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Successfully updated port: bcbc734c-3414-4396-9bc7-a344f91cb894 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1155.515217] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1155.535625] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquiring lock "refresh_cache-a55c8243-3413-435e-89cb-6a2bb65b5292" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1155.535625] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquired lock "refresh_cache-a55c8243-3413-435e-89cb-6a2bb65b5292" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1155.535818] env[63593]: DEBUG nova.network.neutron [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1156.017809] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1156.018121] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1156.018430] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1156.018504] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1156.019520] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25787512-95f8-44fc-a50f-049cf2dfd9e8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1156.029991] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b22bb80b-2655-4cff-9ba4-e1af5daa16e6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1156.045695] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93cb7f9c-7d56-4770-9846-179d0fe31741 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1156.052807] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-926a1c7a-4321-495e-b97c-891d625c6b87 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1156.083209] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181473MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1156.083415] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1156.083563] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1156.085806] env[63593]: DEBUG nova.network.neutron [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1156.261662] env[63593]: DEBUG nova.network.neutron [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Updating instance_info_cache with network_info: [{"id": "bcbc734c-3414-4396-9bc7-a344f91cb894", "address": "fa:16:3e:ce:90:99", "network": {"id": "643b251d-707c-458f-8d6b-7dc4c12e8905", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationTestJSON-364765431-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "6c739264d695443f935d41b24e6b484e", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "55764410-260e-4339-a020-6b30995584bf", "external-id": "nsx-vlan-transportzone-551", "segmentation_id": 551, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbcbc734c-34", "ovs_interfaceid": "bcbc734c-3414-4396-9bc7-a344f91cb894", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1156.764521] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Releasing lock "refresh_cache-a55c8243-3413-435e-89cb-6a2bb65b5292" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1156.764908] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Instance network_info: |[{"id": "bcbc734c-3414-4396-9bc7-a344f91cb894", "address": "fa:16:3e:ce:90:99", "network": {"id": "643b251d-707c-458f-8d6b-7dc4c12e8905", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationTestJSON-364765431-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "6c739264d695443f935d41b24e6b484e", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "55764410-260e-4339-a020-6b30995584bf", "external-id": "nsx-vlan-transportzone-551", "segmentation_id": 551, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbcbc734c-34", "ovs_interfaceid": "bcbc734c-3414-4396-9bc7-a344f91cb894", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1156.765414] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:ce:90:99', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '55764410-260e-4339-a020-6b30995584bf', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'bcbc734c-3414-4396-9bc7-a344f91cb894', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1156.773040] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Creating folder: Project (6c739264d695443f935d41b24e6b484e). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1156.773335] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-be8e516d-ccd3-43b2-9d50-aa935d4b9d20 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1156.786073] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Created folder: Project (6c739264d695443f935d41b24e6b484e) in parent group-v291016. [ 1156.786256] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Creating folder: Instances. Parent ref: group-v291070. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1156.786482] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0b30c356-44af-4b99-a747-3634179293eb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1156.794953] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Created folder: Instances in parent group-v291070. [ 1156.795186] env[63593]: DEBUG oslo.service.loopingcall [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1156.795364] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1156.795553] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3f6d4d86-8478-40ed-9c7c-e6dcb818ea47 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1156.816027] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1156.816027] env[63593]: value = "task-1368020" [ 1156.816027] env[63593]: _type = "Task" [ 1156.816027] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1156.823193] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368020, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1157.045614] env[63593]: DEBUG nova.compute.manager [req-49649edb-0f19-4670-82f6-dc60ff1c8d62 req-a5a402f5-e1f0-4290-92bb-a76d9407b093 service nova] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Received event network-changed-bcbc734c-3414-4396-9bc7-a344f91cb894 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1157.045942] env[63593]: DEBUG nova.compute.manager [req-49649edb-0f19-4670-82f6-dc60ff1c8d62 req-a5a402f5-e1f0-4290-92bb-a76d9407b093 service nova] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Refreshing instance network info cache due to event network-changed-bcbc734c-3414-4396-9bc7-a344f91cb894. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1157.046965] env[63593]: DEBUG oslo_concurrency.lockutils [req-49649edb-0f19-4670-82f6-dc60ff1c8d62 req-a5a402f5-e1f0-4290-92bb-a76d9407b093 service nova] Acquiring lock "refresh_cache-a55c8243-3413-435e-89cb-6a2bb65b5292" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1157.046965] env[63593]: DEBUG oslo_concurrency.lockutils [req-49649edb-0f19-4670-82f6-dc60ff1c8d62 req-a5a402f5-e1f0-4290-92bb-a76d9407b093 service nova] Acquired lock "refresh_cache-a55c8243-3413-435e-89cb-6a2bb65b5292" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1157.046965] env[63593]: DEBUG nova.network.neutron [req-49649edb-0f19-4670-82f6-dc60ff1c8d62 req-a5a402f5-e1f0-4290-92bb-a76d9407b093 service nova] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Refreshing network info cache for port bcbc734c-3414-4396-9bc7-a344f91cb894 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1157.190518] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fbd620a6-3743-4737-98b7-b13928b6587d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.190695] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.190835] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 328b91c6-4ae4-442f-ac87-519658a812ff actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.190954] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.191082] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.191200] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.191314] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.191435] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.191577] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.191700] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1157.325581] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368020, 'name': CreateVM_Task, 'duration_secs': 0.267131} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1157.326161] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1157.326515] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1157.326677] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1157.327058] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1157.327318] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f36572b2-f697-43c4-998c-eb4a877db9d2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1157.331770] env[63593]: DEBUG oslo_vmware.api [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Waiting for the task: (returnval){ [ 1157.331770] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52ad83a2-e494-f794-8d61-f20739c864ad" [ 1157.331770] env[63593]: _type = "Task" [ 1157.331770] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1157.341311] env[63593]: DEBUG oslo_vmware.api [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52ad83a2-e494-f794-8d61-f20739c864ad, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1157.695295] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1157.841536] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1157.841836] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1157.841978] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1157.861855] env[63593]: DEBUG nova.network.neutron [req-49649edb-0f19-4670-82f6-dc60ff1c8d62 req-a5a402f5-e1f0-4290-92bb-a76d9407b093 service nova] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Updated VIF entry in instance network info cache for port bcbc734c-3414-4396-9bc7-a344f91cb894. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1157.862209] env[63593]: DEBUG nova.network.neutron [req-49649edb-0f19-4670-82f6-dc60ff1c8d62 req-a5a402f5-e1f0-4290-92bb-a76d9407b093 service nova] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Updating instance_info_cache with network_info: [{"id": "bcbc734c-3414-4396-9bc7-a344f91cb894", "address": "fa:16:3e:ce:90:99", "network": {"id": "643b251d-707c-458f-8d6b-7dc4c12e8905", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationTestJSON-364765431-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "6c739264d695443f935d41b24e6b484e", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "55764410-260e-4339-a020-6b30995584bf", "external-id": "nsx-vlan-transportzone-551", "segmentation_id": 551, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapbcbc734c-34", "ovs_interfaceid": "bcbc734c-3414-4396-9bc7-a344f91cb894", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1158.198483] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1158.364634] env[63593]: DEBUG oslo_concurrency.lockutils [req-49649edb-0f19-4670-82f6-dc60ff1c8d62 req-a5a402f5-e1f0-4290-92bb-a76d9407b093 service nova] Releasing lock "refresh_cache-a55c8243-3413-435e-89cb-6a2bb65b5292" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1158.701835] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 1f3118b4-f510-48d3-88ba-230564a4e218 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1159.204615] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2a07ea77-f690-4c34-96b2-8ff9ce603326 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1159.708065] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1159.708065] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1159.708065] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1159.724098] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing inventories for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 1159.736857] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Updating ProviderTree inventory for provider 7345f17f-3af2-4b0a-9521-0292dc691877 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 1159.737072] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Updating inventory in ProviderTree for provider 7345f17f-3af2-4b0a-9521-0292dc691877 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1159.746175] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing aggregate associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, aggregates: None {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 1159.762087] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing trait associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 1159.905711] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2eb8369f-107f-4584-b5a7-afeae03b0873 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1159.913067] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27c57d1f-bad7-4283-b859-83fd06f8c898 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1159.942908] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be686eda-b3f5-4cc6-af62-5f4d4354e92e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1159.949742] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05d40040-b4a7-4678-bd0e-ced0281cdb7f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1159.962074] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1160.465837] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1160.971161] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1160.971357] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 4.888s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1163.465541] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1193.636114] env[63593]: WARNING oslo_vmware.rw_handles [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1193.636114] env[63593]: ERROR oslo_vmware.rw_handles [ 1193.636871] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/367d0437-ac08-405b-8669-99d0f4cb480c/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1193.639305] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1193.639485] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Copying Virtual Disk [datastore2] vmware_temp/367d0437-ac08-405b-8669-99d0f4cb480c/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/367d0437-ac08-405b-8669-99d0f4cb480c/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1193.639767] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-c29ae85b-63ae-4700-966f-c1f11131dfb4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1193.649488] env[63593]: DEBUG oslo_vmware.api [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Waiting for the task: (returnval){ [ 1193.649488] env[63593]: value = "task-1368021" [ 1193.649488] env[63593]: _type = "Task" [ 1193.649488] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1193.656993] env[63593]: DEBUG oslo_vmware.api [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Task: {'id': task-1368021, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1194.160070] env[63593]: DEBUG oslo_vmware.exceptions [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1194.160378] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1194.160954] env[63593]: ERROR nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1194.160954] env[63593]: Faults: ['InvalidArgument'] [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Traceback (most recent call last): [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] yield resources [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] self.driver.spawn(context, instance, image_meta, [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] self._fetch_image_if_missing(context, vi) [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] image_cache(vi, tmp_image_ds_loc) [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] vm_util.copy_virtual_disk( [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] session._wait_for_task(vmdk_copy_task) [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] return self.wait_for_task(task_ref) [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] return evt.wait() [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] result = hub.switch() [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] return self.greenlet.switch() [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] self.f(*self.args, **self.kw) [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] raise exceptions.translate_fault(task_info.error) [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Faults: ['InvalidArgument'] [ 1194.160954] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] [ 1194.161982] env[63593]: INFO nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Terminating instance [ 1194.162832] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1194.163074] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1194.163318] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e02a033f-b58e-4ee5-a3ce-ffdea296a911 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.165488] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1194.165683] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1194.166387] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f7caf21-6297-4ba8-b9f2-5166e8bc84f7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.173051] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1194.173263] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-da798946-e58b-435e-b250-0912127d5151 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.175352] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1194.175520] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1194.176433] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-beca36d4-1622-4be5-8b9a-3d498a86bea3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.181070] env[63593]: DEBUG oslo_vmware.api [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 1194.181070] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]524d3788-a48e-1402-40b9-998afa846fc4" [ 1194.181070] env[63593]: _type = "Task" [ 1194.181070] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1194.194846] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1194.195096] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating directory with path [datastore2] vmware_temp/29d47dc8-ffe5-4617-9160-b775ae6e3156/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1194.195301] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-cc03e156-1d6a-46f3-bfe6-b2ecc7b0770b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.216703] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Created directory with path [datastore2] vmware_temp/29d47dc8-ffe5-4617-9160-b775ae6e3156/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1194.216896] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Fetch image to [datastore2] vmware_temp/29d47dc8-ffe5-4617-9160-b775ae6e3156/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1194.217075] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/29d47dc8-ffe5-4617-9160-b775ae6e3156/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1194.217796] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-234b70c1-9e2f-4f17-9779-2389591c7834 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.224361] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf048b9d-928f-435e-bb8e-602e09519455 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.233139] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59d9b185-d0ab-411c-a1a4-eedd541767fa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.264188] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d70edbd-6546-4637-8a85-93069452a8a7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.266565] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1194.266757] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1194.266932] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Deleting the datastore file [datastore2] fbd620a6-3743-4737-98b7-b13928b6587d {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1194.267168] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-dc666f85-02ca-4541-9965-5b3d20158843 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.272413] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-cf3b85d7-4746-4467-a350-ad10f0af1a6c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1194.273990] env[63593]: DEBUG oslo_vmware.api [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Waiting for the task: (returnval){ [ 1194.273990] env[63593]: value = "task-1368023" [ 1194.273990] env[63593]: _type = "Task" [ 1194.273990] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1194.281627] env[63593]: DEBUG oslo_vmware.api [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Task: {'id': task-1368023, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1194.359660] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1194.411487] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/29d47dc8-ffe5-4617-9160-b775ae6e3156/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1194.486479] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1194.486691] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/29d47dc8-ffe5-4617-9160-b775ae6e3156/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1194.785449] env[63593]: DEBUG oslo_vmware.api [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Task: {'id': task-1368023, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.062946} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1194.785749] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1194.785797] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1194.785938] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1194.786103] env[63593]: INFO nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1194.788140] env[63593]: DEBUG nova.compute.claims [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1194.788310] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1194.788515] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1195.451723] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6784429-1fec-416d-af83-cbdd29f1d934 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1195.459452] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f03249e6-e8c8-4de2-b099-1e93c309584e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1195.489301] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ed0c51d-4209-4af4-abf7-dc0740877a8b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1195.496328] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9b8eed8-0d80-4a61-ae84-ff84397003ea {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1195.509366] env[63593]: DEBUG nova.compute.provider_tree [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1196.013613] env[63593]: DEBUG nova.scheduler.client.report [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1196.517679] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.729s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1196.518254] env[63593]: ERROR nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1196.518254] env[63593]: Faults: ['InvalidArgument'] [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Traceback (most recent call last): [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] self.driver.spawn(context, instance, image_meta, [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] self._fetch_image_if_missing(context, vi) [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] image_cache(vi, tmp_image_ds_loc) [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] vm_util.copy_virtual_disk( [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] session._wait_for_task(vmdk_copy_task) [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] return self.wait_for_task(task_ref) [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] return evt.wait() [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] result = hub.switch() [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] return self.greenlet.switch() [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] self.f(*self.args, **self.kw) [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] raise exceptions.translate_fault(task_info.error) [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Faults: ['InvalidArgument'] [ 1196.518254] env[63593]: ERROR nova.compute.manager [instance: fbd620a6-3743-4737-98b7-b13928b6587d] [ 1196.519102] env[63593]: DEBUG nova.compute.utils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1196.521337] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Build of instance fbd620a6-3743-4737-98b7-b13928b6587d was re-scheduled: A specified parameter was not correct: fileType [ 1196.521337] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1196.521454] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1196.521626] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1196.521800] env[63593]: DEBUG nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1196.522076] env[63593]: DEBUG nova.network.neutron [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1197.365309] env[63593]: DEBUG nova.network.neutron [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1197.868214] env[63593]: INFO nova.compute.manager [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Took 1.35 seconds to deallocate network for instance. [ 1198.899866] env[63593]: INFO nova.scheduler.client.report [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Deleted allocations for instance fbd620a6-3743-4737-98b7-b13928b6587d [ 1199.411262] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fe2a4736-fa3b-43cc-bda6-a1e7d3b022fb tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "fbd620a6-3743-4737-98b7-b13928b6587d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 378.774s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1199.411262] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "fbd620a6-3743-4737-98b7-b13928b6587d" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 182.980s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1199.411262] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Acquiring lock "fbd620a6-3743-4737-98b7-b13928b6587d-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1199.411262] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "fbd620a6-3743-4737-98b7-b13928b6587d-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1199.411262] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "fbd620a6-3743-4737-98b7-b13928b6587d-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1199.413142] env[63593]: INFO nova.compute.manager [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Terminating instance [ 1199.414952] env[63593]: DEBUG nova.compute.manager [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1199.415320] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1199.416577] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-712cba9f-d46c-4d76-857d-05b0925bb3ff {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1199.425738] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed54f968-c923-4465-bea8-6238129b28c3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1199.454606] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fbd620a6-3743-4737-98b7-b13928b6587d could not be found. [ 1199.454831] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1199.454991] env[63593]: INFO nova.compute.manager [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1199.455273] env[63593]: DEBUG oslo.service.loopingcall [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1199.455509] env[63593]: DEBUG nova.compute.manager [-] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1199.455594] env[63593]: DEBUG nova.network.neutron [-] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1199.913829] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1199.972881] env[63593]: DEBUG nova.network.neutron [-] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1200.443211] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1200.443421] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1200.444973] env[63593]: INFO nova.compute.claims [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1200.475922] env[63593]: INFO nova.compute.manager [-] [instance: fbd620a6-3743-4737-98b7-b13928b6587d] Took 1.02 seconds to deallocate network for instance. [ 1200.510305] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_power_states {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.019356] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Getting list of instances from cluster (obj){ [ 1201.019356] env[63593]: value = "domain-c8" [ 1201.019356] env[63593]: _type = "ClusterComputeResource" [ 1201.019356] env[63593]: } {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 1201.020137] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97af9006-e48d-43c4-9914-c66201d58f1d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1201.036356] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Got total of 9 instances {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 1201.036578] env[63593]: WARNING nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] While synchronizing instance power states, found 10 instances in the database and 9 instances on the hypervisor. [ 1201.036800] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 268653cf-fcf9-4b50-9234-0b38fb9a2d95 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.036980] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 328b91c6-4ae4-442f-ac87-519658a812ff {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.037191] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid b053676a-81b8-4345-b257-1d67976fd077 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.037383] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid cf724d13-e34e-4a17-9b54-56190891d5db {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.037572] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 48e001ae-cc04-4d95-be43-2b2b3556c6db {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.038277] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.038500] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 4e8f8346-70b0-4ae4-925c-50094ac5a07a {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.038699] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 488c4bca-8279-4de7-8259-238497cbd011 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.039728] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid a55c8243-3413-435e-89cb-6a2bb65b5292 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.039983] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 41e38c8a-3aa1-4524-8820-27ba85187fe8 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1201.040440] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.040741] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "328b91c6-4ae4-442f-ac87-519658a812ff" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.040998] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "b053676a-81b8-4345-b257-1d67976fd077" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.041262] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "cf724d13-e34e-4a17-9b54-56190891d5db" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.041509] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "48e001ae-cc04-4d95-be43-2b2b3556c6db" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.041776] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.042065] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.042373] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "488c4bca-8279-4de7-8259-238497cbd011" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.042676] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "a55c8243-3413-435e-89cb-6a2bb65b5292" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.042961] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "41e38c8a-3aa1-4524-8820-27ba85187fe8" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1201.508488] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bbca5a29-1f4e-4ed3-bb13-9bef19f14662 tempest-VolumesAdminNegativeTest-1997284620 tempest-VolumesAdminNegativeTest-1997284620-project-member] Lock "fbd620a6-3743-4737-98b7-b13928b6587d" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.098s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1201.671341] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f26934cc-78d6-4e46-b18c-558c2f5d8fca {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1201.679018] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9677681f-63ee-4bf9-a410-c5794f1e7649 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1201.708332] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc0be409-8c4f-4f10-abfe-8f55af8f37bd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1201.715625] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-865fff34-ca39-43d7-94e9-746dd3e3a721 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1201.729656] env[63593]: DEBUG nova.compute.provider_tree [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1202.233153] env[63593]: DEBUG nova.scheduler.client.report [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1202.740920] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.297s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1202.741514] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1203.248268] env[63593]: DEBUG nova.compute.utils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1203.249665] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1203.249831] env[63593]: DEBUG nova.network.neutron [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1203.291823] env[63593]: DEBUG nova.policy [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a6da77eb2136460cbdda8b8272f33139', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c6358000debb48aaa7e831652b061e2d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1203.620344] env[63593]: DEBUG nova.network.neutron [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Successfully created port: 9ea5f94e-e785-4896-9060-a762ffe3ef95 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1203.753089] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1204.764699] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1204.790655] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1204.790896] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1204.791066] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1204.791250] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1204.791393] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1204.791536] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1204.791967] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1204.792272] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1204.792501] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1204.792722] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1204.792949] env[63593]: DEBUG nova.virt.hardware [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1204.793865] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-903daad3-5d21-4788-afc3-0895537af119 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1204.802396] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a5de875-40da-44dc-ac1f-67973d5c2af3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1205.470229] env[63593]: DEBUG nova.compute.manager [req-1f37f415-b5c6-4d64-8fce-b819cb1e750f req-4e2bacc3-4d23-4e9f-88af-a6a11b9f6304 service nova] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Received event network-vif-plugged-9ea5f94e-e785-4896-9060-a762ffe3ef95 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1205.470539] env[63593]: DEBUG oslo_concurrency.lockutils [req-1f37f415-b5c6-4d64-8fce-b819cb1e750f req-4e2bacc3-4d23-4e9f-88af-a6a11b9f6304 service nova] Acquiring lock "41e38c8a-3aa1-4524-8820-27ba85187fe8-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1205.472114] env[63593]: DEBUG oslo_concurrency.lockutils [req-1f37f415-b5c6-4d64-8fce-b819cb1e750f req-4e2bacc3-4d23-4e9f-88af-a6a11b9f6304 service nova] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1205.472114] env[63593]: DEBUG oslo_concurrency.lockutils [req-1f37f415-b5c6-4d64-8fce-b819cb1e750f req-4e2bacc3-4d23-4e9f-88af-a6a11b9f6304 service nova] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1205.472114] env[63593]: DEBUG nova.compute.manager [req-1f37f415-b5c6-4d64-8fce-b819cb1e750f req-4e2bacc3-4d23-4e9f-88af-a6a11b9f6304 service nova] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] No waiting events found dispatching network-vif-plugged-9ea5f94e-e785-4896-9060-a762ffe3ef95 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1205.472114] env[63593]: WARNING nova.compute.manager [req-1f37f415-b5c6-4d64-8fce-b819cb1e750f req-4e2bacc3-4d23-4e9f-88af-a6a11b9f6304 service nova] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Received unexpected event network-vif-plugged-9ea5f94e-e785-4896-9060-a762ffe3ef95 for instance with vm_state building and task_state spawning. [ 1205.584696] env[63593]: DEBUG nova.network.neutron [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Successfully updated port: 9ea5f94e-e785-4896-9060-a762ffe3ef95 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1206.086665] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquiring lock "refresh_cache-41e38c8a-3aa1-4524-8820-27ba85187fe8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1206.086957] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquired lock "refresh_cache-41e38c8a-3aa1-4524-8820-27ba85187fe8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1206.086957] env[63593]: DEBUG nova.network.neutron [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1206.633426] env[63593]: DEBUG nova.network.neutron [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1206.788034] env[63593]: DEBUG nova.network.neutron [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Updating instance_info_cache with network_info: [{"id": "9ea5f94e-e785-4896-9060-a762ffe3ef95", "address": "fa:16:3e:f4:ba:bf", "network": {"id": "940261d6-02fa-4df3-9f77-b39cab5e952e", "bridge": "br-int", "label": "tempest-AttachVolumeShelveTestJSON-421922410-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c6358000debb48aaa7e831652b061e2d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "04ebd8af-aaf6-4d04-b869-3882e2571ed7", "external-id": "nsx-vlan-transportzone-541", "segmentation_id": 541, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ea5f94e-e7", "ovs_interfaceid": "9ea5f94e-e785-4896-9060-a762ffe3ef95", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1207.239293] env[63593]: DEBUG oslo_concurrency.lockutils [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquiring lock "488c4bca-8279-4de7-8259-238497cbd011" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1207.294210] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Releasing lock "refresh_cache-41e38c8a-3aa1-4524-8820-27ba85187fe8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1207.294210] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Instance network_info: |[{"id": "9ea5f94e-e785-4896-9060-a762ffe3ef95", "address": "fa:16:3e:f4:ba:bf", "network": {"id": "940261d6-02fa-4df3-9f77-b39cab5e952e", "bridge": "br-int", "label": "tempest-AttachVolumeShelveTestJSON-421922410-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c6358000debb48aaa7e831652b061e2d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "04ebd8af-aaf6-4d04-b869-3882e2571ed7", "external-id": "nsx-vlan-transportzone-541", "segmentation_id": 541, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ea5f94e-e7", "ovs_interfaceid": "9ea5f94e-e785-4896-9060-a762ffe3ef95", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1207.294210] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:f4:ba:bf', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '04ebd8af-aaf6-4d04-b869-3882e2571ed7', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '9ea5f94e-e785-4896-9060-a762ffe3ef95', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1207.301104] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Creating folder: Project (c6358000debb48aaa7e831652b061e2d). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1207.301870] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-8acf3382-ebcb-434d-bf4a-f26381753ec8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1207.313201] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Created folder: Project (c6358000debb48aaa7e831652b061e2d) in parent group-v291016. [ 1207.313417] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Creating folder: Instances. Parent ref: group-v291073. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1207.313657] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-54be2848-ed7e-43c5-972a-6bb6323c2a25 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1207.323280] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Created folder: Instances in parent group-v291073. [ 1207.323545] env[63593]: DEBUG oslo.service.loopingcall [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1207.323760] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1207.323982] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-6af55f2c-683d-4eee-823b-eab2dd859d06 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1207.343352] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1207.343352] env[63593]: value = "task-1368026" [ 1207.343352] env[63593]: _type = "Task" [ 1207.343352] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1207.350547] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368026, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1207.613367] env[63593]: DEBUG nova.compute.manager [req-451343db-0e9c-452c-b1cb-70e5ee5a60fc req-fd86cc00-cc72-4cab-aafb-6bdc563428a5 service nova] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Received event network-changed-9ea5f94e-e785-4896-9060-a762ffe3ef95 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1207.613367] env[63593]: DEBUG nova.compute.manager [req-451343db-0e9c-452c-b1cb-70e5ee5a60fc req-fd86cc00-cc72-4cab-aafb-6bdc563428a5 service nova] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Refreshing instance network info cache due to event network-changed-9ea5f94e-e785-4896-9060-a762ffe3ef95. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1207.613630] env[63593]: DEBUG oslo_concurrency.lockutils [req-451343db-0e9c-452c-b1cb-70e5ee5a60fc req-fd86cc00-cc72-4cab-aafb-6bdc563428a5 service nova] Acquiring lock "refresh_cache-41e38c8a-3aa1-4524-8820-27ba85187fe8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1207.613630] env[63593]: DEBUG oslo_concurrency.lockutils [req-451343db-0e9c-452c-b1cb-70e5ee5a60fc req-fd86cc00-cc72-4cab-aafb-6bdc563428a5 service nova] Acquired lock "refresh_cache-41e38c8a-3aa1-4524-8820-27ba85187fe8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1207.613773] env[63593]: DEBUG nova.network.neutron [req-451343db-0e9c-452c-b1cb-70e5ee5a60fc req-fd86cc00-cc72-4cab-aafb-6bdc563428a5 service nova] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Refreshing network info cache for port 9ea5f94e-e785-4896-9060-a762ffe3ef95 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1207.853252] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368026, 'name': CreateVM_Task, 'duration_secs': 0.345983} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1207.853424] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1207.854116] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1207.854285] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1207.854626] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1207.854869] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5d9db77b-a765-466b-95df-b0bdee9b49d8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1207.859093] env[63593]: DEBUG oslo_vmware.api [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Waiting for the task: (returnval){ [ 1207.859093] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5278eaeb-ccb7-62e5-d19b-4308fb5a1dc0" [ 1207.859093] env[63593]: _type = "Task" [ 1207.859093] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1207.866249] env[63593]: DEBUG oslo_vmware.api [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5278eaeb-ccb7-62e5-d19b-4308fb5a1dc0, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1208.136121] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquiring lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1208.136329] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1208.349813] env[63593]: DEBUG nova.network.neutron [req-451343db-0e9c-452c-b1cb-70e5ee5a60fc req-fd86cc00-cc72-4cab-aafb-6bdc563428a5 service nova] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Updated VIF entry in instance network info cache for port 9ea5f94e-e785-4896-9060-a762ffe3ef95. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1208.350383] env[63593]: DEBUG nova.network.neutron [req-451343db-0e9c-452c-b1cb-70e5ee5a60fc req-fd86cc00-cc72-4cab-aafb-6bdc563428a5 service nova] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Updating instance_info_cache with network_info: [{"id": "9ea5f94e-e785-4896-9060-a762ffe3ef95", "address": "fa:16:3e:f4:ba:bf", "network": {"id": "940261d6-02fa-4df3-9f77-b39cab5e952e", "bridge": "br-int", "label": "tempest-AttachVolumeShelveTestJSON-421922410-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c6358000debb48aaa7e831652b061e2d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "04ebd8af-aaf6-4d04-b869-3882e2571ed7", "external-id": "nsx-vlan-transportzone-541", "segmentation_id": 541, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9ea5f94e-e7", "ovs_interfaceid": "9ea5f94e-e785-4896-9060-a762ffe3ef95", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1208.369461] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1208.369721] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1208.369926] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1208.853693] env[63593]: DEBUG oslo_concurrency.lockutils [req-451343db-0e9c-452c-b1cb-70e5ee5a60fc req-fd86cc00-cc72-4cab-aafb-6bdc563428a5 service nova] Releasing lock "refresh_cache-41e38c8a-3aa1-4524-8820-27ba85187fe8" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1210.013159] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1210.013516] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1213.013638] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1213.013916] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1213.013958] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1213.519371] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.519539] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.519722] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b053676a-81b8-4345-b257-1d67976fd077] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.519874] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.520017] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.520138] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.520258] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.520374] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.520609] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.520609] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1213.520720] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1213.520935] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1214.012620] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1214.012847] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1214.013023] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1216.008569] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1216.012373] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1217.013155] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1217.523621] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1217.523974] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1217.524106] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1217.524453] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1217.525435] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-531c2723-2414-49c9-b730-3fe939f98a4d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1217.535247] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13fe987b-fee8-449f-9f16-733e250e4d27 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1217.552369] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a8872d1-644a-4d97-87c9-30537733bb64 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1217.560261] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c7c4996-fe44-4853-8882-787bdda799bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1217.589643] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181458MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1217.589861] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1217.590025] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1217.601139] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1217.601395] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1218.623546] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1218.623817] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 328b91c6-4ae4-442f-ac87-519658a812ff actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1218.623817] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1218.623960] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1218.624042] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1218.624157] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1218.624271] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1218.624382] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1218.624502] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1218.624724] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1219.127812] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1219.631457] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 1f3118b4-f510-48d3-88ba-230564a4e218 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1220.136900] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2a07ea77-f690-4c34-96b2-8ff9ce603326 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1220.640104] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1220.735872] env[63593]: DEBUG oslo_concurrency.lockutils [None req-1ef9715f-539f-4eda-b157-38bfee7302ea tempest-ServerPasswordTestJSON-463763678 tempest-ServerPasswordTestJSON-463763678-project-member] Acquiring lock "ab5cdb92-e2de-48d9-9abe-9c46b43a89f6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1220.736138] env[63593]: DEBUG oslo_concurrency.lockutils [None req-1ef9715f-539f-4eda-b157-38bfee7302ea tempest-ServerPasswordTestJSON-463763678 tempest-ServerPasswordTestJSON-463763678-project-member] Lock "ab5cdb92-e2de-48d9-9abe-9c46b43a89f6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1221.142956] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1221.646826] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1221.647250] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1221.647250] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1221.850052] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44a2c800-0776-4e3a-a9ef-2754a660b075 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1221.857900] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b74b8d9-2c9f-4686-bbdf-f574d30ff101 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1221.887818] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4747684-cda8-40e5-8b57-9daaaf77874c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1221.895750] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50415592-fa82-488f-b00e-d75856faa073 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1221.909548] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1222.412578] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1222.917853] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1222.918200] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 5.328s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1223.097687] env[63593]: DEBUG oslo_concurrency.lockutils [None req-44d03183-2d52-46c3-9c5d-59c364c1adef tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "8d041272-16e4-4e87-95b0-62c5e3c9db2f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1223.098014] env[63593]: DEBUG oslo_concurrency.lockutils [None req-44d03183-2d52-46c3-9c5d-59c364c1adef tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "8d041272-16e4-4e87-95b0-62c5e3c9db2f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1230.295316] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquiring lock "a55c8243-3413-435e-89cb-6a2bb65b5292" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1242.567664] env[63593]: WARNING oslo_vmware.rw_handles [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1242.567664] env[63593]: ERROR oslo_vmware.rw_handles [ 1242.568313] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/29d47dc8-ffe5-4617-9160-b775ae6e3156/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1242.570186] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1242.570437] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Copying Virtual Disk [datastore2] vmware_temp/29d47dc8-ffe5-4617-9160-b775ae6e3156/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/29d47dc8-ffe5-4617-9160-b775ae6e3156/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1242.570726] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-1f627d3a-9fea-4ecc-9ec0-db3d9d274c73 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1242.578647] env[63593]: DEBUG oslo_vmware.api [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 1242.578647] env[63593]: value = "task-1368027" [ 1242.578647] env[63593]: _type = "Task" [ 1242.578647] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1242.586576] env[63593]: DEBUG oslo_vmware.api [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': task-1368027, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1243.088347] env[63593]: DEBUG oslo_vmware.exceptions [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1243.088679] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1243.089293] env[63593]: ERROR nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1243.089293] env[63593]: Faults: ['InvalidArgument'] [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Traceback (most recent call last): [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] yield resources [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] self.driver.spawn(context, instance, image_meta, [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] self._fetch_image_if_missing(context, vi) [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] image_cache(vi, tmp_image_ds_loc) [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] vm_util.copy_virtual_disk( [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] session._wait_for_task(vmdk_copy_task) [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] return self.wait_for_task(task_ref) [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] return evt.wait() [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] result = hub.switch() [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] return self.greenlet.switch() [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] self.f(*self.args, **self.kw) [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] raise exceptions.translate_fault(task_info.error) [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Faults: ['InvalidArgument'] [ 1243.089293] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] [ 1243.090357] env[63593]: INFO nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Terminating instance [ 1243.091320] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1243.091561] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1243.091800] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-79d01e95-349b-4fcb-8a9f-b4d60589c0a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.093951] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1243.094155] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1243.094868] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a1cea3e-43dd-40d6-b304-6c1ee87c3eed {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.101432] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1243.101638] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-3546a2e8-0ac2-4b80-bd95-5d8f7403b6c8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.103658] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1243.103822] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1243.104721] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-9cef77ea-556b-4231-ae03-c2ae70b6b164 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.109147] env[63593]: DEBUG oslo_vmware.api [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Waiting for the task: (returnval){ [ 1243.109147] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52132a35-ac1a-44f4-7993-757e111bebe9" [ 1243.109147] env[63593]: _type = "Task" [ 1243.109147] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1243.125048] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1243.125048] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Creating directory with path [datastore2] vmware_temp/56c6f196-0e2a-4ba8-be09-1fe4c73aa979/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1243.125048] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8a2b6b53-38cb-49a7-beb8-e8617f1e73d4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.143808] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Created directory with path [datastore2] vmware_temp/56c6f196-0e2a-4ba8-be09-1fe4c73aa979/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1243.143999] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Fetch image to [datastore2] vmware_temp/56c6f196-0e2a-4ba8-be09-1fe4c73aa979/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1243.144182] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/56c6f196-0e2a-4ba8-be09-1fe4c73aa979/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1243.144942] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-330d3424-6131-461d-a618-5a64897d466c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.151347] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60eef200-8950-4823-83bf-06019fb16e94 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.160436] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a38fc54-cb54-42b5-b73f-3e8ea00789f4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.191305] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a07f5dd7-8d80-4e65-a875-ed39a9f3ce3d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.194952] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1243.195174] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1243.195346] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Deleting the datastore file [datastore2] 268653cf-fcf9-4b50-9234-0b38fb9a2d95 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1243.195934] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-429ccf74-2ee4-4809-aef0-f879aff62eb7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.199266] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-afa0c2a9-12dd-424b-b764-527613a69793 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.203496] env[63593]: DEBUG oslo_vmware.api [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 1243.203496] env[63593]: value = "task-1368029" [ 1243.203496] env[63593]: _type = "Task" [ 1243.203496] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1243.210575] env[63593]: DEBUG oslo_vmware.api [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': task-1368029, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1243.219689] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1243.269053] env[63593]: DEBUG oslo_vmware.rw_handles [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/56c6f196-0e2a-4ba8-be09-1fe4c73aa979/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1243.330557] env[63593]: DEBUG oslo_vmware.rw_handles [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1243.330734] env[63593]: DEBUG oslo_vmware.rw_handles [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/56c6f196-0e2a-4ba8-be09-1fe4c73aa979/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1243.713349] env[63593]: DEBUG oslo_vmware.api [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': task-1368029, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.069637} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1243.713685] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1243.713787] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1243.713957] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1243.714139] env[63593]: INFO nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1243.716269] env[63593]: DEBUG nova.compute.claims [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1243.716442] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1243.716650] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1244.407426] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24469410-19f0-4f97-a60e-7868ec341c14 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.415136] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-153fd01d-cc67-4a09-bd92-c5645b0ab5a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.445069] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e43de4e9-ed87-4ffb-a369-06c6dd7fca3a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.452091] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ffdcfa9-9ede-4d9f-80b0-a3e9d03e3b57 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.464766] env[63593]: DEBUG nova.compute.provider_tree [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1244.967794] env[63593]: DEBUG nova.scheduler.client.report [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1245.473254] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.756s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1245.473925] env[63593]: ERROR nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1245.473925] env[63593]: Faults: ['InvalidArgument'] [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Traceback (most recent call last): [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] self.driver.spawn(context, instance, image_meta, [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] self._fetch_image_if_missing(context, vi) [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] image_cache(vi, tmp_image_ds_loc) [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] vm_util.copy_virtual_disk( [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] session._wait_for_task(vmdk_copy_task) [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] return self.wait_for_task(task_ref) [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] return evt.wait() [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] result = hub.switch() [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] return self.greenlet.switch() [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] self.f(*self.args, **self.kw) [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] raise exceptions.translate_fault(task_info.error) [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Faults: ['InvalidArgument'] [ 1245.473925] env[63593]: ERROR nova.compute.manager [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] [ 1245.474941] env[63593]: DEBUG nova.compute.utils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1245.476429] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Build of instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 was re-scheduled: A specified parameter was not correct: fileType [ 1245.476429] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1245.476799] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1245.476969] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1245.477148] env[63593]: DEBUG nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1245.477310] env[63593]: DEBUG nova.network.neutron [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1246.241430] env[63593]: DEBUG nova.network.neutron [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1246.744482] env[63593]: INFO nova.compute.manager [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Took 1.27 seconds to deallocate network for instance. [ 1247.779007] env[63593]: INFO nova.scheduler.client.report [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Deleted allocations for instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 [ 1248.288342] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2422b31c-6d38-4103-8e43-a00fc98d72fd tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 425.030s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1248.288726] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 228.574s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1248.288883] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1248.289107] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1248.289278] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1248.291832] env[63593]: INFO nova.compute.manager [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Terminating instance [ 1248.293727] env[63593]: DEBUG nova.compute.manager [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1248.293981] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1248.294194] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-777be8b0-7e2c-426d-aea4-402771a5bb92 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1248.303674] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f4f9328-7ed8-48ff-83a4-6827a4b08233 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1248.330905] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 268653cf-fcf9-4b50-9234-0b38fb9a2d95 could not be found. [ 1248.331131] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1248.332028] env[63593]: INFO nova.compute.manager [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1248.332028] env[63593]: DEBUG oslo.service.loopingcall [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1248.332028] env[63593]: DEBUG nova.compute.manager [-] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1248.332028] env[63593]: DEBUG nova.network.neutron [-] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1248.793055] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1248.852662] env[63593]: DEBUG nova.network.neutron [-] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1249.310278] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1249.310527] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1249.312035] env[63593]: INFO nova.compute.claims [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1249.355125] env[63593]: INFO nova.compute.manager [-] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] Took 1.02 seconds to deallocate network for instance. [ 1250.379238] env[63593]: DEBUG oslo_concurrency.lockutils [None req-34ea5668-b138-43b0-b1e5-c8730a445076 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.091s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1250.380144] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 49.340s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1250.380512] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 268653cf-fcf9-4b50-9234-0b38fb9a2d95] During sync_power_state the instance has a pending task (deleting). Skip. [ 1250.380512] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "268653cf-fcf9-4b50-9234-0b38fb9a2d95" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1250.519174] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0f8581c-e0b0-42a6-b29e-c31a1bcf6d20 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1250.527590] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5003c9b4-c7f6-43ff-86a4-88eb60b0af6e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1250.558550] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-166da08b-40bc-4b5c-8e65-1440d22352b9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1250.567693] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d47e01dc-85f4-4d6c-ac32-1ca8758628d2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1250.585286] env[63593]: DEBUG nova.compute.provider_tree [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1251.088921] env[63593]: DEBUG nova.scheduler.client.report [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1251.594094] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.283s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1251.594545] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1252.099410] env[63593]: DEBUG nova.compute.utils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1252.100800] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1252.100974] env[63593]: DEBUG nova.network.neutron [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1252.142726] env[63593]: DEBUG nova.policy [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3bb930b4d1534f3ea3abde038aa60100', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3101ac6769714328b1b7e2c5f371bccc', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1252.449185] env[63593]: DEBUG nova.network.neutron [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Successfully created port: 97ab3eb0-048c-4d48-a8a1-9ab29d93583a {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1252.604559] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1253.616398] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1253.641079] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1253.641341] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1253.641499] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1253.641678] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1253.641824] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1253.641968] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1253.642275] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1253.642452] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1253.642708] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1253.642873] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1253.643054] env[63593]: DEBUG nova.virt.hardware [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1253.644036] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dbc46248-d5ca-40f3-b2b6-09e50221583b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1253.652580] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f8dae93-a516-44e7-bd69-37ecd8487205 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1254.335926] env[63593]: DEBUG nova.compute.manager [req-ae3a31e3-9741-463a-b12f-9d9fc46319bf req-acc9e9cc-1cb4-4327-ae21-9d966880eed9 service nova] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Received event network-vif-plugged-97ab3eb0-048c-4d48-a8a1-9ab29d93583a {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1254.336166] env[63593]: DEBUG oslo_concurrency.lockutils [req-ae3a31e3-9741-463a-b12f-9d9fc46319bf req-acc9e9cc-1cb4-4327-ae21-9d966880eed9 service nova] Acquiring lock "442bcb9d-533f-4072-b967-791e3bc99643-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1254.336379] env[63593]: DEBUG oslo_concurrency.lockutils [req-ae3a31e3-9741-463a-b12f-9d9fc46319bf req-acc9e9cc-1cb4-4327-ae21-9d966880eed9 service nova] Lock "442bcb9d-533f-4072-b967-791e3bc99643-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1254.336545] env[63593]: DEBUG oslo_concurrency.lockutils [req-ae3a31e3-9741-463a-b12f-9d9fc46319bf req-acc9e9cc-1cb4-4327-ae21-9d966880eed9 service nova] Lock "442bcb9d-533f-4072-b967-791e3bc99643-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1254.336712] env[63593]: DEBUG nova.compute.manager [req-ae3a31e3-9741-463a-b12f-9d9fc46319bf req-acc9e9cc-1cb4-4327-ae21-9d966880eed9 service nova] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] No waiting events found dispatching network-vif-plugged-97ab3eb0-048c-4d48-a8a1-9ab29d93583a {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1254.336891] env[63593]: WARNING nova.compute.manager [req-ae3a31e3-9741-463a-b12f-9d9fc46319bf req-acc9e9cc-1cb4-4327-ae21-9d966880eed9 service nova] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Received unexpected event network-vif-plugged-97ab3eb0-048c-4d48-a8a1-9ab29d93583a for instance with vm_state building and task_state spawning. [ 1254.890728] env[63593]: DEBUG nova.network.neutron [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Successfully updated port: 97ab3eb0-048c-4d48-a8a1-9ab29d93583a {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1254.913236] env[63593]: DEBUG nova.compute.manager [req-18d1befb-514c-40fe-9231-aaacf7a745d0 req-dd267113-0530-465a-acab-6b9d0800f1d7 service nova] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Received event network-changed-97ab3eb0-048c-4d48-a8a1-9ab29d93583a {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1254.913396] env[63593]: DEBUG nova.compute.manager [req-18d1befb-514c-40fe-9231-aaacf7a745d0 req-dd267113-0530-465a-acab-6b9d0800f1d7 service nova] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Refreshing instance network info cache due to event network-changed-97ab3eb0-048c-4d48-a8a1-9ab29d93583a. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1254.913476] env[63593]: DEBUG oslo_concurrency.lockutils [req-18d1befb-514c-40fe-9231-aaacf7a745d0 req-dd267113-0530-465a-acab-6b9d0800f1d7 service nova] Acquiring lock "refresh_cache-442bcb9d-533f-4072-b967-791e3bc99643" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1254.913722] env[63593]: DEBUG oslo_concurrency.lockutils [req-18d1befb-514c-40fe-9231-aaacf7a745d0 req-dd267113-0530-465a-acab-6b9d0800f1d7 service nova] Acquired lock "refresh_cache-442bcb9d-533f-4072-b967-791e3bc99643" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1254.913898] env[63593]: DEBUG nova.network.neutron [req-18d1befb-514c-40fe-9231-aaacf7a745d0 req-dd267113-0530-465a-acab-6b9d0800f1d7 service nova] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Refreshing network info cache for port 97ab3eb0-048c-4d48-a8a1-9ab29d93583a {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1255.395126] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "refresh_cache-442bcb9d-533f-4072-b967-791e3bc99643" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1255.444874] env[63593]: DEBUG nova.network.neutron [req-18d1befb-514c-40fe-9231-aaacf7a745d0 req-dd267113-0530-465a-acab-6b9d0800f1d7 service nova] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1255.556015] env[63593]: DEBUG nova.network.neutron [req-18d1befb-514c-40fe-9231-aaacf7a745d0 req-dd267113-0530-465a-acab-6b9d0800f1d7 service nova] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1256.058486] env[63593]: DEBUG oslo_concurrency.lockutils [req-18d1befb-514c-40fe-9231-aaacf7a745d0 req-dd267113-0530-465a-acab-6b9d0800f1d7 service nova] Releasing lock "refresh_cache-442bcb9d-533f-4072-b967-791e3bc99643" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1256.058857] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "refresh_cache-442bcb9d-533f-4072-b967-791e3bc99643" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1256.059042] env[63593]: DEBUG nova.network.neutron [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1256.590494] env[63593]: DEBUG nova.network.neutron [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1256.726876] env[63593]: DEBUG nova.network.neutron [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Updating instance_info_cache with network_info: [{"id": "97ab3eb0-048c-4d48-a8a1-9ab29d93583a", "address": "fa:16:3e:e3:7c:a9", "network": {"id": "aca8be2a-21d8-4636-8216-f46c0892ddc8", "bridge": "br-int", "label": "tempest-ServersTestJSON-848574930-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3101ac6769714328b1b7e2c5f371bccc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap97ab3eb0-04", "ovs_interfaceid": "97ab3eb0-048c-4d48-a8a1-9ab29d93583a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1257.229235] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "refresh_cache-442bcb9d-533f-4072-b967-791e3bc99643" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1257.229575] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Instance network_info: |[{"id": "97ab3eb0-048c-4d48-a8a1-9ab29d93583a", "address": "fa:16:3e:e3:7c:a9", "network": {"id": "aca8be2a-21d8-4636-8216-f46c0892ddc8", "bridge": "br-int", "label": "tempest-ServersTestJSON-848574930-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3101ac6769714328b1b7e2c5f371bccc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap97ab3eb0-04", "ovs_interfaceid": "97ab3eb0-048c-4d48-a8a1-9ab29d93583a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1257.229993] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:e3:7c:a9', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '7d377d75-3add-4a15-8691-74b2eb010924', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '97ab3eb0-048c-4d48-a8a1-9ab29d93583a', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1257.237242] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating folder: Project (3101ac6769714328b1b7e2c5f371bccc). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1257.237503] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-8a1634ea-6df7-4017-8be6-523a302cce3c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1257.249155] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Created folder: Project (3101ac6769714328b1b7e2c5f371bccc) in parent group-v291016. [ 1257.249330] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating folder: Instances. Parent ref: group-v291076. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1257.249541] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6a23c251-0932-42fe-914b-65def1b4b707 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1257.260107] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Created folder: Instances in parent group-v291076. [ 1257.260313] env[63593]: DEBUG oslo.service.loopingcall [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1257.260495] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1257.260687] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-bd891510-bd06-465e-9dd5-c6fc2e50b3a9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1257.279017] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1257.279017] env[63593]: value = "task-1368032" [ 1257.279017] env[63593]: _type = "Task" [ 1257.279017] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1257.286030] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368032, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1257.789052] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368032, 'name': CreateVM_Task, 'duration_secs': 0.450124} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1257.789052] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1257.789251] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1257.789338] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1257.789633] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1257.789882] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-cc51d0d8-d1bb-450c-b2c1-29c199f23b0a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1257.794090] env[63593]: DEBUG oslo_vmware.api [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 1257.794090] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]528dfc05-cee0-893f-a535-5fc4476007af" [ 1257.794090] env[63593]: _type = "Task" [ 1257.794090] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1257.801981] env[63593]: DEBUG oslo_vmware.api [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]528dfc05-cee0-893f-a535-5fc4476007af, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1258.304147] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1258.304427] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1258.304600] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1261.388346] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "a1b46566-4db2-4086-906a-7ba32b9a2813" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1261.388346] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1272.545643] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "442bcb9d-533f-4072-b967-791e3bc99643" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1272.790429] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquiring lock "41e38c8a-3aa1-4524-8820-27ba85187fe8" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1277.918969] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1277.919245] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1277.919386] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1277.919501] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1278.423927] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.424127] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b053676a-81b8-4345-b257-1d67976fd077] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.424260] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.424386] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.424507] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.424625] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.424739] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.424855] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.424974] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.425109] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1278.425228] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1278.425426] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1278.425583] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1278.425727] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1278.425871] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1278.426048] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1278.426212] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1278.426338] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1278.426477] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1278.929920] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1278.930220] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1278.930359] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1278.930510] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1278.931467] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fff46d2c-60f8-44fa-8fd1-eaccfcd4b411 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1278.939917] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb326c5e-3be2-41c6-9998-7391d4a87b3b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1278.954278] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0ed3a6a-c987-4fcb-bade-bf7f5ab0a5dd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1278.964866] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-934c3043-0701-4069-944c-79264627f274 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1278.992048] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181452MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1278.992227] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1278.992434] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1280.024808] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 328b91c6-4ae4-442f-ac87-519658a812ff actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.024808] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.025107] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.025107] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.025168] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.025313] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.025453] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.025567] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.025678] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.025785] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1280.528989] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2a07ea77-f690-4c34-96b2-8ff9ce603326 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1281.034327] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1281.537356] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1282.042930] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1282.546749] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance ab5cdb92-e2de-48d9-9abe-9c46b43a89f6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1283.051448] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 8d041272-16e4-4e87-95b0-62c5e3c9db2f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1283.555879] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1283.556162] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1283.556301] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1283.810062] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29e0ce0e-f9a7-4b0c-84a0-a734d96f5ddf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.820743] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94b67578-419a-4bbe-8602-d4d6347947ca {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.848562] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87ea1fff-7efe-4d1c-a5e8-3fe09c0f89c9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.856012] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3529e1b2-67e3-494f-97e3-d7a7c6d77db3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.868607] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1284.070810] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cefb2407-ed33-43ef-8b98-dfcc34f61a05 tempest-AttachInterfacesUnderV243Test-729252606 tempest-AttachInterfacesUnderV243Test-729252606-project-member] Acquiring lock "e4d93e31-6b0e-4720-bf2a-40794ff79308" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1284.071427] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cefb2407-ed33-43ef-8b98-dfcc34f61a05 tempest-AttachInterfacesUnderV243Test-729252606 tempest-AttachInterfacesUnderV243Test-729252606-project-member] Lock "e4d93e31-6b0e-4720-bf2a-40794ff79308" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1284.375022] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1284.880573] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1284.880800] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 5.888s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1285.971859] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1286.149936] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a79d8d6c-928d-4efb-ba68-a2cf1949de57 tempest-ServersAaction247Test-1718206514 tempest-ServersAaction247Test-1718206514-project-member] Acquiring lock "28999e3f-02b8-4cb8-adbf-192dc86cb565" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1286.149936] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a79d8d6c-928d-4efb-ba68-a2cf1949de57 tempest-ServersAaction247Test-1718206514 tempest-ServersAaction247Test-1718206514-project-member] Lock "28999e3f-02b8-4cb8-adbf-192dc86cb565" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1288.735944] env[63593]: WARNING oslo_vmware.rw_handles [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1288.735944] env[63593]: ERROR oslo_vmware.rw_handles [ 1288.736512] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/56c6f196-0e2a-4ba8-be09-1fe4c73aa979/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1288.738554] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1288.738797] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Copying Virtual Disk [datastore2] vmware_temp/56c6f196-0e2a-4ba8-be09-1fe4c73aa979/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/56c6f196-0e2a-4ba8-be09-1fe4c73aa979/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1288.739089] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-21595cbd-5aae-4041-aeb2-40a17f71bad9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1288.747108] env[63593]: DEBUG oslo_vmware.api [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Waiting for the task: (returnval){ [ 1288.747108] env[63593]: value = "task-1368033" [ 1288.747108] env[63593]: _type = "Task" [ 1288.747108] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1288.755462] env[63593]: DEBUG oslo_vmware.api [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Task: {'id': task-1368033, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1289.257334] env[63593]: DEBUG oslo_vmware.exceptions [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1289.257620] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1289.258170] env[63593]: ERROR nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1289.258170] env[63593]: Faults: ['InvalidArgument'] [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Traceback (most recent call last): [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] yield resources [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self.driver.spawn(context, instance, image_meta, [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._fetch_image_if_missing(context, vi) [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] image_cache(vi, tmp_image_ds_loc) [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] vm_util.copy_virtual_disk( [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] session._wait_for_task(vmdk_copy_task) [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.wait_for_task(task_ref) [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return evt.wait() [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] result = hub.switch() [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.greenlet.switch() [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self.f(*self.args, **self.kw) [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] raise exceptions.translate_fault(task_info.error) [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Faults: ['InvalidArgument'] [ 1289.258170] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1289.258834] env[63593]: INFO nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Terminating instance [ 1289.260077] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1289.260286] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1289.260888] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1289.261092] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1289.261312] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-91b6cdd6-0066-4ac6-906a-0df71ac06e98 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.263714] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f79ca5bd-c545-41e6-8ae4-2dbd28f3f446 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.270686] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1289.270974] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-d5ba333f-11eb-4b57-a234-9bab0ae9a121 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.273048] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1289.273246] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1289.274215] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-64d93e7d-e096-4bc5-8c5a-bdce823140c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.278824] env[63593]: DEBUG oslo_vmware.api [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for the task: (returnval){ [ 1289.278824] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52b4e54e-8a69-9628-efb2-f7972f0a6f68" [ 1289.278824] env[63593]: _type = "Task" [ 1289.278824] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1289.286890] env[63593]: DEBUG oslo_vmware.api [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52b4e54e-8a69-9628-efb2-f7972f0a6f68, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1289.377923] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1289.377923] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1289.377923] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Deleting the datastore file [datastore2] 328b91c6-4ae4-442f-ac87-519658a812ff {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1289.377923] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-a6e8127e-4065-4d83-9226-21628e93b981 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.382647] env[63593]: DEBUG oslo_vmware.api [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Waiting for the task: (returnval){ [ 1289.382647] env[63593]: value = "task-1368035" [ 1289.382647] env[63593]: _type = "Task" [ 1289.382647] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1289.390293] env[63593]: DEBUG oslo_vmware.api [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Task: {'id': task-1368035, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1289.789463] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1289.789755] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Creating directory with path [datastore2] vmware_temp/38e06ac4-a790-41a2-9754-3ff178809731/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1289.789951] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b77d3725-0a66-44e8-b9ca-25844a17ab62 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.801156] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Created directory with path [datastore2] vmware_temp/38e06ac4-a790-41a2-9754-3ff178809731/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1289.801357] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Fetch image to [datastore2] vmware_temp/38e06ac4-a790-41a2-9754-3ff178809731/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1289.801522] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/38e06ac4-a790-41a2-9754-3ff178809731/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1289.802250] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca044f25-7643-4a01-a978-b1f2ad83bfe5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.808777] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7043cc0e-0f5e-4491-8f50-7e42c89a65d7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.817361] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87630d3c-088f-4cb9-8d2f-89959c2a32d5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.847412] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4676522-c8fe-4e21-bf0d-ed9eb88ecfd6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.852742] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-9262ec67-9edd-435b-9b38-f8d52b7228d1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1289.871388] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1289.891022] env[63593]: DEBUG oslo_vmware.api [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Task: {'id': task-1368035, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.073417} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1289.892814] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1289.893034] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1289.893218] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1289.893423] env[63593]: INFO nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Took 0.63 seconds to destroy the instance on the hypervisor. [ 1289.897298] env[63593]: DEBUG nova.compute.claims [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1289.897298] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1289.897298] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1289.925860] env[63593]: DEBUG oslo_vmware.rw_handles [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/38e06ac4-a790-41a2-9754-3ff178809731/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1289.986817] env[63593]: DEBUG oslo_vmware.rw_handles [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1289.987054] env[63593]: DEBUG oslo_vmware.rw_handles [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/38e06ac4-a790-41a2-9754-3ff178809731/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1290.591921] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d573e8f-a041-4b05-8d5e-dc683a7e8182 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1290.598985] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f73eb37d-6d56-4a40-bc2d-cc72fbda9a8e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1290.626880] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-587e21bc-789d-4fcb-bdd3-27a61f8266c7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1290.633330] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a337ba8-4cbd-45d3-bc0a-1710ce8b4bcb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1290.645567] env[63593]: DEBUG nova.compute.provider_tree [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1291.148312] env[63593]: DEBUG nova.scheduler.client.report [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1291.654095] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.756s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1291.654095] env[63593]: ERROR nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1291.654095] env[63593]: Faults: ['InvalidArgument'] [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Traceback (most recent call last): [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self.driver.spawn(context, instance, image_meta, [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._fetch_image_if_missing(context, vi) [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] image_cache(vi, tmp_image_ds_loc) [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] vm_util.copy_virtual_disk( [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] session._wait_for_task(vmdk_copy_task) [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.wait_for_task(task_ref) [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return evt.wait() [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] result = hub.switch() [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.greenlet.switch() [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self.f(*self.args, **self.kw) [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] raise exceptions.translate_fault(task_info.error) [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Faults: ['InvalidArgument'] [ 1291.654095] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1291.655039] env[63593]: DEBUG nova.compute.utils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1291.656301] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Build of instance 328b91c6-4ae4-442f-ac87-519658a812ff was re-scheduled: A specified parameter was not correct: fileType [ 1291.656301] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1291.656655] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1291.656821] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1291.656978] env[63593]: DEBUG nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1291.657154] env[63593]: DEBUG nova.network.neutron [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1291.797263] env[63593]: DEBUG neutronclient.v2_0.client [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63593) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1291.798717] env[63593]: ERROR nova.compute.manager [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Failed to deallocate networks: nova.exception.Unauthorized: Not authorized. [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Traceback (most recent call last): [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self.driver.spawn(context, instance, image_meta, [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._fetch_image_if_missing(context, vi) [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] image_cache(vi, tmp_image_ds_loc) [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] vm_util.copy_virtual_disk( [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] session._wait_for_task(vmdk_copy_task) [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.wait_for_task(task_ref) [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return evt.wait() [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] result = hub.switch() [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.greenlet.switch() [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self.f(*self.args, **self.kw) [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] raise exceptions.translate_fault(task_info.error) [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Faults: ['InvalidArgument'] [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] During handling of the above exception, another exception occurred: [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Traceback (most recent call last): [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 2456, in _do_build_and_run_instance [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._build_and_run_instance(context, instance, image, [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 2748, in _build_and_run_instance [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] raise exception.RescheduledException( [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] nova.exception.RescheduledException: Build of instance 328b91c6-4ae4-442f-ac87-519658a812ff was re-scheduled: A specified parameter was not correct: fileType [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Faults: ['InvalidArgument'] [ 1291.798717] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] During handling of the above exception, another exception occurred: [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Traceback (most recent call last): [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] exception_handler_v20(status_code, error_body) [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] raise client_exc(message=error_message, [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Neutron server returns request_ids: ['req-b65b456f-dc62-4cae-a850-0fd367d4d8a9'] [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] During handling of the above exception, another exception occurred: [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Traceback (most recent call last): [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 3045, in _cleanup_allocated_networks [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._deallocate_network(context, instance, requested_networks) [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 2291, in _deallocate_network [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self.network_api.deallocate_for_instance( [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] data = neutron.list_ports(**search_opts) [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.list('ports', self.ports_path, retrieve_all, [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] for r in self._pagination(collection, path, **params): [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] res = self.get(path, params=params) [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.retry_request("GET", action, body=body, [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1291.799712] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.do_request(method, action, body=body, [ 1291.800764] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1291.800764] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1291.800764] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1291.800764] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._handle_fault_response(status_code, replybody, resp) [ 1291.800764] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 204, in wrapper [ 1291.800764] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] raise exception.Unauthorized() [ 1291.800764] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] nova.exception.Unauthorized: Not authorized. [ 1291.800764] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1292.327280] env[63593]: INFO nova.scheduler.client.report [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Deleted allocations for instance 328b91c6-4ae4-442f-ac87-519658a812ff [ 1292.838104] env[63593]: DEBUG oslo_concurrency.lockutils [None req-7c930dac-4c61-4ddd-b85a-e2bb84054f93 tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "328b91c6-4ae4-442f-ac87-519658a812ff" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 468.796s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1292.839751] env[63593]: DEBUG oslo_concurrency.lockutils [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "328b91c6-4ae4-442f-ac87-519658a812ff" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 272.798s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1292.839944] env[63593]: DEBUG oslo_concurrency.lockutils [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Acquiring lock "328b91c6-4ae4-442f-ac87-519658a812ff-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1292.840170] env[63593]: DEBUG oslo_concurrency.lockutils [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "328b91c6-4ae4-442f-ac87-519658a812ff-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1292.840340] env[63593]: DEBUG oslo_concurrency.lockutils [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "328b91c6-4ae4-442f-ac87-519658a812ff-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1292.842107] env[63593]: INFO nova.compute.manager [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Terminating instance [ 1292.843747] env[63593]: DEBUG nova.compute.manager [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1292.843984] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1292.844291] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2da1a42e-c9d4-403b-acbe-5e19aaef953b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1292.853242] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6c6bb03-a9d2-4220-9a09-bb59eec295a9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1292.878668] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 328b91c6-4ae4-442f-ac87-519658a812ff could not be found. [ 1292.878855] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1292.879037] env[63593]: INFO nova.compute.manager [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1292.879269] env[63593]: DEBUG oslo.service.loopingcall [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1292.879483] env[63593]: DEBUG nova.compute.manager [-] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1292.879574] env[63593]: DEBUG nova.network.neutron [-] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1292.963106] env[63593]: DEBUG neutronclient.v2_0.client [-] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63593) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1292.963394] env[63593]: ERROR nova.network.neutron [-] Neutron client was not able to generate a valid admin token, please verify Neutron admin credential located in nova.conf: neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall [-] Dynamic interval looping call 'oslo_service.loopingcall.RetryDecorator.__call__.._func' failed: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall exception_handler_v20(status_code, error_body) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall raise client_exc(message=error_message, [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall Neutron server returns request_ids: ['req-d4eba4e5-2f37-424f-89c8-74e501afe8bf'] [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall During handling of the above exception, another exception occurred: [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall result = func(*self.args, **self.kw) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall result = f(*args, **kwargs) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 3071, in _deallocate_network_with_retries [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall self._deallocate_network( [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 2291, in _deallocate_network [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall self.network_api.deallocate_for_instance( [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall data = neutron.list_ports(**search_opts) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall return self.list('ports', self.ports_path, retrieve_all, [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall for r in self._pagination(collection, path, **params): [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall res = self.get(path, params=params) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall return self.retry_request("GET", action, body=body, [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall return self.do_request(method, action, body=body, [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall self._handle_fault_response(status_code, replybody, resp) [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1292.964097] env[63593]: ERROR oslo.service.loopingcall [ 1292.965395] env[63593]: ERROR nova.compute.manager [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Failed to deallocate network for instance. Error: Networking client is experiencing an unauthorized exception.: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1293.343860] env[63593]: DEBUG nova.compute.manager [None req-9aaaa7d9-378b-46a7-8590-1cf9d49bfb11 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 1f3118b4-f510-48d3-88ba-230564a4e218] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1293.469724] env[63593]: ERROR nova.compute.manager [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Setting instance vm_state to ERROR: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Traceback (most recent call last): [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] exception_handler_v20(status_code, error_body) [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] raise client_exc(message=error_message, [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Neutron server returns request_ids: ['req-d4eba4e5-2f37-424f-89c8-74e501afe8bf'] [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] During handling of the above exception, another exception occurred: [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Traceback (most recent call last): [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 3341, in do_terminate_instance [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._delete_instance(context, instance, bdms) [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 3276, in _delete_instance [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._shutdown_instance(context, instance, bdms) [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 3170, in _shutdown_instance [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._try_deallocate_network(context, instance, requested_networks) [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 3084, in _try_deallocate_network [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] with excutils.save_and_reraise_exception(): [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self.force_reraise() [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] raise self.value [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 3082, in _try_deallocate_network [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] _deallocate_network_with_retries() [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return evt.wait() [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] result = hub.switch() [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.greenlet.switch() [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] result = func(*self.args, **self.kw) [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1293.469724] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] result = f(*args, **kwargs) [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 3071, in _deallocate_network_with_retries [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._deallocate_network( [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/compute/manager.py", line 2291, in _deallocate_network [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self.network_api.deallocate_for_instance( [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] data = neutron.list_ports(**search_opts) [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.list('ports', self.ports_path, retrieve_all, [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] for r in self._pagination(collection, path, **params): [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] res = self.get(path, params=params) [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.retry_request("GET", action, body=body, [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] return self.do_request(method, action, body=body, [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] ret = obj(*args, **kwargs) [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] self._handle_fault_response(status_code, replybody, resp) [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1293.470812] env[63593]: ERROR nova.compute.manager [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] [ 1293.847312] env[63593]: DEBUG nova.compute.manager [None req-9aaaa7d9-378b-46a7-8590-1cf9d49bfb11 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] [instance: 1f3118b4-f510-48d3-88ba-230564a4e218] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1293.974243] env[63593]: DEBUG oslo_concurrency.lockutils [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Lock "328b91c6-4ae4-442f-ac87-519658a812ff" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 1.134s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1293.975480] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "328b91c6-4ae4-442f-ac87-519658a812ff" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 92.935s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1293.975675] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] During sync_power_state the instance has a pending task (deleting). Skip. [ 1293.975864] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "328b91c6-4ae4-442f-ac87-519658a812ff" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1294.361933] env[63593]: DEBUG oslo_concurrency.lockutils [None req-9aaaa7d9-378b-46a7-8590-1cf9d49bfb11 tempest-AttachInterfacesTestJSON-1698632525 tempest-AttachInterfacesTestJSON-1698632525-project-member] Lock "1f3118b4-f510-48d3-88ba-230564a4e218" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 212.793s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1294.864876] env[63593]: DEBUG nova.compute.manager [None req-e370cb6c-3042-4545-adf6-0f1d039085f5 tempest-ImagesNegativeTestJSON-1494673914 tempest-ImagesNegativeTestJSON-1494673914-project-member] [instance: 2a07ea77-f690-4c34-96b2-8ff9ce603326] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1295.368707] env[63593]: DEBUG nova.compute.manager [None req-e370cb6c-3042-4545-adf6-0f1d039085f5 tempest-ImagesNegativeTestJSON-1494673914 tempest-ImagesNegativeTestJSON-1494673914-project-member] [instance: 2a07ea77-f690-4c34-96b2-8ff9ce603326] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1295.494240] env[63593]: INFO nova.compute.manager [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] [instance: 328b91c6-4ae4-442f-ac87-519658a812ff] Successfully reverted task state from None on failure for instance. [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server [None req-32d64eb7-0064-4a58-b0c1-339ac11957df tempest-ServersTestMultiNic-1904013270 tempest-ServersTestMultiNic-1904013270-project-member] Exception during message handling: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server exception_handler_v20(status_code, error_body) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server raise client_exc(message=error_message, [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server Neutron server returns request_ids: ['req-d4eba4e5-2f37-424f-89c8-74e501afe8bf'] [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/server.py", line 172, in _process_incoming [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 166, in decorated_function [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 157, in decorated_function [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/utils.py", line 1453, in decorated_function [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 213, in decorated_function [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 203, in decorated_function [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3353, in terminate_instance [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server do_terminate_instance(instance, bdms) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py", line 412, in inner [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server return f(*args, **kwargs) [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3348, in do_terminate_instance [ 1295.499329] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3341, in do_terminate_instance [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server self._delete_instance(context, instance, bdms) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3276, in _delete_instance [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server self._shutdown_instance(context, instance, bdms) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3170, in _shutdown_instance [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server self._try_deallocate_network(context, instance, requested_networks) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3084, in _try_deallocate_network [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3082, in _try_deallocate_network [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server _deallocate_network_with_retries() [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server return evt.wait() [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server result = hub.switch() [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server return self.greenlet.switch() [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server result = func(*self.args, **self.kw) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server result = f(*args, **kwargs) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3071, in _deallocate_network_with_retries [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server self._deallocate_network( [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2291, in _deallocate_network [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server self.network_api.deallocate_for_instance( [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server data = neutron.list_ports(**search_opts) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server return self.list('ports', self.ports_path, retrieve_all, [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server for r in self._pagination(collection, path, **params): [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server res = self.get(path, params=params) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server return self.retry_request("GET", action, body=body, [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server return self.do_request(method, action, body=body, [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1295.500758] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1295.502162] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1295.502162] env[63593]: ERROR oslo_messaging.rpc.server self._handle_fault_response(status_code, replybody, resp) [ 1295.502162] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1295.502162] env[63593]: ERROR oslo_messaging.rpc.server raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1295.502162] env[63593]: ERROR oslo_messaging.rpc.server nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1295.502162] env[63593]: ERROR oslo_messaging.rpc.server [ 1295.881298] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e370cb6c-3042-4545-adf6-0f1d039085f5 tempest-ImagesNegativeTestJSON-1494673914 tempest-ImagesNegativeTestJSON-1494673914-project-member] Lock "2a07ea77-f690-4c34-96b2-8ff9ce603326" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 208.676s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1296.384130] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1296.904038] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1296.904302] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1296.905774] env[63593]: INFO nova.compute.claims [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1298.106500] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1345d806-ed7c-4016-999d-13dc6c98c2e5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1298.114089] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6255b82d-386d-4cb3-9f94-0dcc57910bd3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1298.143493] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e416e50e-f6e5-4447-a6d4-d8c812827083 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1298.150264] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f10cc95-cb60-4c62-b7db-b972554d6574 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1298.163139] env[63593]: DEBUG nova.compute.provider_tree [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1298.667127] env[63593]: DEBUG nova.scheduler.client.report [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1299.171269] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.267s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1299.171776] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1299.676434] env[63593]: DEBUG nova.compute.utils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1299.677794] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Not allocating networking since 'none' was specified. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 1300.180225] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1301.189407] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1301.213782] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1301.214023] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1301.214203] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1301.214383] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1301.214525] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1301.214667] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1301.214867] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1301.215031] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1301.215200] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1301.215360] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1301.215525] env[63593]: DEBUG nova.virt.hardware [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1301.216382] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8598a07e-f63e-413a-8565-060ee7f0246d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1301.224558] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc14beac-9bbd-41d4-bd2c-6c022109753c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1301.238469] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Instance VIF info [] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1301.243788] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Creating folder: Project (f3e162f729ef485fa865e0f447b50cc3). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1301.244153] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-60773f86-5693-427d-ad2c-fc45704499a0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1301.253122] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Created folder: Project (f3e162f729ef485fa865e0f447b50cc3) in parent group-v291016. [ 1301.253302] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Creating folder: Instances. Parent ref: group-v291079. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1301.253504] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b60a0134-d8da-4284-8a13-a64c172d13c3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1301.260883] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Created folder: Instances in parent group-v291079. [ 1301.261173] env[63593]: DEBUG oslo.service.loopingcall [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1301.261366] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1301.261588] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-e62b0008-9876-48a3-83f7-a7f9ed2adf44 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1301.276629] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1301.276629] env[63593]: value = "task-1368038" [ 1301.276629] env[63593]: _type = "Task" [ 1301.276629] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1301.283331] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368038, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1301.787063] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368038, 'name': CreateVM_Task, 'duration_secs': 0.234413} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1301.787063] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1301.787511] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1301.787674] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1301.787997] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1301.788260] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-66a5416f-ff8f-4f5a-960b-e5750088082e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1301.792487] env[63593]: DEBUG oslo_vmware.api [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Waiting for the task: (returnval){ [ 1301.792487] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5209f663-44a4-ae39-aaa3-8e419a50ac71" [ 1301.792487] env[63593]: _type = "Task" [ 1301.792487] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1301.799724] env[63593]: DEBUG oslo_vmware.api [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5209f663-44a4-ae39-aaa3-8e419a50ac71, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1302.302431] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1302.302718] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1302.302980] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1332.013446] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1332.013741] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1333.013532] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1333.013890] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1333.013890] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1333.518868] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b053676a-81b8-4345-b257-1d67976fd077] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.518868] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.518868] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.518868] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.519078] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.519078] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.519136] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.519881] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.519881] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.519881] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1333.519881] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1336.012886] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1336.013146] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1336.013299] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1337.604210] env[63593]: WARNING oslo_vmware.rw_handles [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1337.604210] env[63593]: ERROR oslo_vmware.rw_handles [ 1337.604751] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/38e06ac4-a790-41a2-9754-3ff178809731/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1337.606749] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1337.606987] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Copying Virtual Disk [datastore2] vmware_temp/38e06ac4-a790-41a2-9754-3ff178809731/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/38e06ac4-a790-41a2-9754-3ff178809731/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1337.607282] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-e1095dbf-eea7-4888-9fb6-223d64328c59 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1337.615746] env[63593]: DEBUG oslo_vmware.api [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for the task: (returnval){ [ 1337.615746] env[63593]: value = "task-1368039" [ 1337.615746] env[63593]: _type = "Task" [ 1337.615746] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1337.623055] env[63593]: DEBUG oslo_vmware.api [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': task-1368039, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1338.008184] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1338.011865] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1338.012071] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1338.124901] env[63593]: DEBUG oslo_vmware.exceptions [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1338.125190] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1338.125747] env[63593]: ERROR nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1338.125747] env[63593]: Faults: ['InvalidArgument'] [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] Traceback (most recent call last): [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] yield resources [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] self.driver.spawn(context, instance, image_meta, [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] self._fetch_image_if_missing(context, vi) [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] image_cache(vi, tmp_image_ds_loc) [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] vm_util.copy_virtual_disk( [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] session._wait_for_task(vmdk_copy_task) [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] return self.wait_for_task(task_ref) [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] return evt.wait() [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] result = hub.switch() [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] return self.greenlet.switch() [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] self.f(*self.args, **self.kw) [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] raise exceptions.translate_fault(task_info.error) [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] Faults: ['InvalidArgument'] [ 1338.125747] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] [ 1338.126640] env[63593]: INFO nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Terminating instance [ 1338.127581] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1338.128299] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1338.128925] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1338.129118] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1338.129338] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0fe88f06-24dc-4522-91b3-aa9edd4f9429 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.131470] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4607ae76-58eb-44c5-a889-b65452337a1e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.138951] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1338.139992] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-ad5926e7-560b-40f8-b8bd-9c827e578a06 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.141252] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1338.141417] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1338.142083] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-abb9086e-37e0-4ba7-ba74-d0e320da7a44 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.147343] env[63593]: DEBUG oslo_vmware.api [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Waiting for the task: (returnval){ [ 1338.147343] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5292882b-0504-0ab2-bd20-d70972ec92b6" [ 1338.147343] env[63593]: _type = "Task" [ 1338.147343] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1338.153948] env[63593]: DEBUG oslo_vmware.api [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5292882b-0504-0ab2-bd20-d70972ec92b6, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1338.208234] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1338.208453] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1338.208635] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Deleting the datastore file [datastore2] b053676a-81b8-4345-b257-1d67976fd077 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1338.208911] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-637c5d12-8eaa-4b52-8a1e-3d2034c80ca6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.215276] env[63593]: DEBUG oslo_vmware.api [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for the task: (returnval){ [ 1338.215276] env[63593]: value = "task-1368041" [ 1338.215276] env[63593]: _type = "Task" [ 1338.215276] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1338.222526] env[63593]: DEBUG oslo_vmware.api [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': task-1368041, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1338.657412] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1338.657686] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Creating directory with path [datastore2] vmware_temp/c02030e0-b3cc-487d-840e-ace077af2990/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1338.657914] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-de6820e8-1fcc-44c7-a49e-4c2e8c26d604 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.669881] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Created directory with path [datastore2] vmware_temp/c02030e0-b3cc-487d-840e-ace077af2990/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1338.670088] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Fetch image to [datastore2] vmware_temp/c02030e0-b3cc-487d-840e-ace077af2990/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1338.670257] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/c02030e0-b3cc-487d-840e-ace077af2990/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1338.670957] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c973905-b53e-4a83-9b0b-b68d7dd47d9a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.677023] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2e0e045-301f-42c6-8b4f-29751ebe1d5f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.686488] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19a6ea8b-5226-4f0c-9572-5911547a3c65 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.723566] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f503070-ba3f-4545-82f9-658e49e8b6aa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.731057] env[63593]: DEBUG oslo_vmware.api [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': task-1368041, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.080776} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1338.732471] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1338.732657] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1338.732824] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1338.732991] env[63593]: INFO nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1338.734755] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-a8600129-77be-40da-9d7c-20b0810dc3a4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1338.736951] env[63593]: DEBUG nova.compute.claims [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1338.737137] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1338.737350] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1338.764440] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1338.811948] env[63593]: DEBUG oslo_vmware.rw_handles [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c02030e0-b3cc-487d-840e-ace077af2990/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1338.873112] env[63593]: DEBUG oslo_vmware.rw_handles [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1338.873246] env[63593]: DEBUG oslo_vmware.rw_handles [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c02030e0-b3cc-487d-840e-ace077af2990/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1339.012587] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1339.432391] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5bae871b-5093-4d40-86ed-eed431ad7bda {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1339.439983] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5a5e9f2-acf3-4a7a-b6e7-0a13c0c17cc5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1339.470426] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3308acc-4392-4de9-977a-80b651f047fe {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1339.477596] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3e6579c-c48a-42bf-ac08-972d4bcd2534 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1339.490437] env[63593]: DEBUG nova.compute.provider_tree [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1339.516009] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1339.993613] env[63593]: DEBUG nova.scheduler.client.report [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1340.498358] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.761s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1340.498941] env[63593]: ERROR nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1340.498941] env[63593]: Faults: ['InvalidArgument'] [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] Traceback (most recent call last): [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] self.driver.spawn(context, instance, image_meta, [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] self._fetch_image_if_missing(context, vi) [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] image_cache(vi, tmp_image_ds_loc) [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] vm_util.copy_virtual_disk( [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] session._wait_for_task(vmdk_copy_task) [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] return self.wait_for_task(task_ref) [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] return evt.wait() [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] result = hub.switch() [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] return self.greenlet.switch() [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] self.f(*self.args, **self.kw) [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] raise exceptions.translate_fault(task_info.error) [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] Faults: ['InvalidArgument'] [ 1340.498941] env[63593]: ERROR nova.compute.manager [instance: b053676a-81b8-4345-b257-1d67976fd077] [ 1340.499807] env[63593]: DEBUG nova.compute.utils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1340.500859] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.985s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1340.501056] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1340.501211] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1340.501707] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Build of instance b053676a-81b8-4345-b257-1d67976fd077 was re-scheduled: A specified parameter was not correct: fileType [ 1340.501707] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1340.502091] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1340.502261] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1340.502426] env[63593]: DEBUG nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1340.502584] env[63593]: DEBUG nova.network.neutron [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1340.504858] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6240afd-938e-4bfc-9099-8138d3d19c55 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1340.512955] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b63aa72a-b49c-41e5-8416-f9713c58a101 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1340.526951] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12e5cf35-c15d-4368-9623-e10f28f9ec54 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1340.533569] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c592621-aa3f-4216-93bd-859628f5cd33 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1340.563943] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181470MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1340.564101] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1340.564307] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1341.268943] env[63593]: DEBUG nova.network.neutron [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1341.771979] env[63593]: INFO nova.compute.manager [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Took 1.27 seconds to deallocate network for instance. [ 1342.109790] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance b053676a-81b8-4345-b257-1d67976fd077 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1342.110099] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance cf724d13-e34e-4a17-9b54-56190891d5db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1342.110310] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1342.110499] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1342.110683] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1342.110864] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1342.111054] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1342.111247] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1342.111427] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1342.111607] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1342.615588] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1342.811725] env[63593]: INFO nova.scheduler.client.report [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Deleted allocations for instance b053676a-81b8-4345-b257-1d67976fd077 [ 1343.118904] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1343.319638] env[63593]: DEBUG oslo_concurrency.lockutils [None req-119179bd-004f-44ce-a100-dc0fdff53497 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "b053676a-81b8-4345-b257-1d67976fd077" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 512.660s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1343.320975] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "b053676a-81b8-4345-b257-1d67976fd077" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 317.610s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1343.321224] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "b053676a-81b8-4345-b257-1d67976fd077-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1343.321437] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "b053676a-81b8-4345-b257-1d67976fd077-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1343.321622] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "b053676a-81b8-4345-b257-1d67976fd077-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1343.323482] env[63593]: INFO nova.compute.manager [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Terminating instance [ 1343.325245] env[63593]: DEBUG nova.compute.manager [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1343.325435] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1343.325730] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f8659a04-dfe1-41ec-b26c-fc084569087e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1343.335102] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74073855-e869-4028-8390-72d30f9f9499 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1343.363135] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b053676a-81b8-4345-b257-1d67976fd077 could not be found. [ 1343.363338] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1343.363569] env[63593]: INFO nova.compute.manager [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: b053676a-81b8-4345-b257-1d67976fd077] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1343.363806] env[63593]: DEBUG oslo.service.loopingcall [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1343.364039] env[63593]: DEBUG nova.compute.manager [-] [instance: b053676a-81b8-4345-b257-1d67976fd077] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1343.364135] env[63593]: DEBUG nova.network.neutron [-] [instance: b053676a-81b8-4345-b257-1d67976fd077] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1343.621496] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance ab5cdb92-e2de-48d9-9abe-9c46b43a89f6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1343.825061] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1343.882439] env[63593]: DEBUG nova.network.neutron [-] [instance: b053676a-81b8-4345-b257-1d67976fd077] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1344.125181] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 8d041272-16e4-4e87-95b0-62c5e3c9db2f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1344.345825] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1344.385067] env[63593]: INFO nova.compute.manager [-] [instance: b053676a-81b8-4345-b257-1d67976fd077] Took 1.02 seconds to deallocate network for instance. [ 1344.628901] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1345.131979] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance e4d93e31-6b0e-4720-bf2a-40794ff79308 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1345.407719] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0cc5eddb-92b0-4bf6-876e-d66420dc0232 tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "b053676a-81b8-4345-b257-1d67976fd077" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.087s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1345.408690] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "b053676a-81b8-4345-b257-1d67976fd077" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 144.368s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1345.408895] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: b053676a-81b8-4345-b257-1d67976fd077] During sync_power_state the instance has a pending task (deleting). Skip. [ 1345.409073] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "b053676a-81b8-4345-b257-1d67976fd077" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1345.634868] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 28999e3f-02b8-4cb8-adbf-192dc86cb565 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1345.635177] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 9 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1345.635281] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2240MB phys_disk=200GB used_disk=9GB total_vcpus=48 used_vcpus=9 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1345.796957] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1fea2076-a1e9-43b1-9a5b-dead7389b2c8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1345.804223] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50cf137d-e861-41eb-ac21-18b1baa6164b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1345.834184] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e7633f5-7c1d-413e-8bb6-bb7de30231fa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1345.840880] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db848bcf-59e0-46a2-b0ed-d227ea935c0a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1345.853163] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1346.356509] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1346.861788] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1346.862063] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 6.298s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1346.862289] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 2.517s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1346.863919] env[63593]: INFO nova.compute.claims [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1348.037552] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "34867c6c-b8ae-401b-b6f4-ce36bea956ae" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1348.048499] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35030a50-aebc-4757-81c2-f39eca24671b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1348.055988] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dd0d40a-d144-4e21-8eeb-c12ea89a14c4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1348.086205] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e90f659-10e8-446b-a2df-e657ab781b11 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1348.092907] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9dc0c6e-c21b-41ef-a031-f47b491387b3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1348.105322] env[63593]: DEBUG nova.compute.provider_tree [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1348.608344] env[63593]: DEBUG nova.scheduler.client.report [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1349.114324] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.252s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1349.114876] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1349.619970] env[63593]: DEBUG nova.compute.utils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1349.621354] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1349.621523] env[63593]: DEBUG nova.network.neutron [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1349.661175] env[63593]: DEBUG nova.policy [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7714bfe721cd427895f8f7bcedf70fab', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cc0483ca6f804e17a88376fe2ced2ec7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1350.038794] env[63593]: DEBUG nova.network.neutron [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Successfully created port: 923a100b-6f79-4c51-97f2-aff1c3cf9ddc {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1350.125610] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1351.136527] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1351.160577] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1351.160810] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1351.160963] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1351.161151] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1351.161292] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1351.161431] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1351.161632] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1351.161785] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1351.161946] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1351.162165] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1351.162285] env[63593]: DEBUG nova.virt.hardware [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1351.163153] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b65be850-2fb9-4290-afe7-12c8d58b90be {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1351.171226] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45ac151f-9c99-4fa2-907b-7f11b315bd62 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1351.614938] env[63593]: DEBUG nova.compute.manager [req-f0e741c0-6118-4cf3-b14d-69872cf3498c req-8588cb0c-8ce9-440e-beed-ae724a024883 service nova] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Received event network-vif-plugged-923a100b-6f79-4c51-97f2-aff1c3cf9ddc {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1351.615181] env[63593]: DEBUG oslo_concurrency.lockutils [req-f0e741c0-6118-4cf3-b14d-69872cf3498c req-8588cb0c-8ce9-440e-beed-ae724a024883 service nova] Acquiring lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1351.615388] env[63593]: DEBUG oslo_concurrency.lockutils [req-f0e741c0-6118-4cf3-b14d-69872cf3498c req-8588cb0c-8ce9-440e-beed-ae724a024883 service nova] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1351.615552] env[63593]: DEBUG oslo_concurrency.lockutils [req-f0e741c0-6118-4cf3-b14d-69872cf3498c req-8588cb0c-8ce9-440e-beed-ae724a024883 service nova] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1351.615760] env[63593]: DEBUG nova.compute.manager [req-f0e741c0-6118-4cf3-b14d-69872cf3498c req-8588cb0c-8ce9-440e-beed-ae724a024883 service nova] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] No waiting events found dispatching network-vif-plugged-923a100b-6f79-4c51-97f2-aff1c3cf9ddc {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1351.616375] env[63593]: WARNING nova.compute.manager [req-f0e741c0-6118-4cf3-b14d-69872cf3498c req-8588cb0c-8ce9-440e-beed-ae724a024883 service nova] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Received unexpected event network-vif-plugged-923a100b-6f79-4c51-97f2-aff1c3cf9ddc for instance with vm_state building and task_state spawning. [ 1351.706340] env[63593]: DEBUG nova.network.neutron [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Successfully updated port: 923a100b-6f79-4c51-97f2-aff1c3cf9ddc {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1352.208628] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquiring lock "refresh_cache-fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1352.208893] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquired lock "refresh_cache-fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1352.208938] env[63593]: DEBUG nova.network.neutron [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1352.742783] env[63593]: DEBUG nova.network.neutron [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1352.873839] env[63593]: DEBUG nova.network.neutron [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Updating instance_info_cache with network_info: [{"id": "923a100b-6f79-4c51-97f2-aff1c3cf9ddc", "address": "fa:16:3e:62:11:c4", "network": {"id": "6216d3ac-5759-45dd-98ae-f61087681a52", "bridge": "br-int", "label": "tempest-ServersTestJSON-919761004-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "cc0483ca6f804e17a88376fe2ced2ec7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dc6d5964-1106-4345-a26d-185dabd4ff0f", "external-id": "nsx-vlan-transportzone-603", "segmentation_id": 603, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap923a100b-6f", "ovs_interfaceid": "923a100b-6f79-4c51-97f2-aff1c3cf9ddc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1353.377234] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Releasing lock "refresh_cache-fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1353.377548] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Instance network_info: |[{"id": "923a100b-6f79-4c51-97f2-aff1c3cf9ddc", "address": "fa:16:3e:62:11:c4", "network": {"id": "6216d3ac-5759-45dd-98ae-f61087681a52", "bridge": "br-int", "label": "tempest-ServersTestJSON-919761004-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "cc0483ca6f804e17a88376fe2ced2ec7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dc6d5964-1106-4345-a26d-185dabd4ff0f", "external-id": "nsx-vlan-transportzone-603", "segmentation_id": 603, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap923a100b-6f", "ovs_interfaceid": "923a100b-6f79-4c51-97f2-aff1c3cf9ddc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1353.377969] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:62:11:c4', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'dc6d5964-1106-4345-a26d-185dabd4ff0f', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '923a100b-6f79-4c51-97f2-aff1c3cf9ddc', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1353.385303] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Creating folder: Project (cc0483ca6f804e17a88376fe2ced2ec7). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1353.385600] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f83d03ff-bdad-433c-87c3-288f1b11a401 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1353.398183] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Created folder: Project (cc0483ca6f804e17a88376fe2ced2ec7) in parent group-v291016. [ 1353.398373] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Creating folder: Instances. Parent ref: group-v291082. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1353.398601] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-8a3a8a88-ae4e-4f75-a3b0-b774cc640acc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1353.407208] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Created folder: Instances in parent group-v291082. [ 1353.407428] env[63593]: DEBUG oslo.service.loopingcall [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1353.407604] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1353.407790] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3e923ee4-60ff-44d6-ba26-eb907a3cfc0d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1353.424807] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1353.424807] env[63593]: value = "task-1368044" [ 1353.424807] env[63593]: _type = "Task" [ 1353.424807] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1353.432058] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368044, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1353.640017] env[63593]: DEBUG nova.compute.manager [req-097b81ef-a0b5-4d8d-9c97-21e76ee36567 req-4f1b95a8-f3a7-47c5-9254-950b924f453e service nova] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Received event network-changed-923a100b-6f79-4c51-97f2-aff1c3cf9ddc {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1353.640156] env[63593]: DEBUG nova.compute.manager [req-097b81ef-a0b5-4d8d-9c97-21e76ee36567 req-4f1b95a8-f3a7-47c5-9254-950b924f453e service nova] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Refreshing instance network info cache due to event network-changed-923a100b-6f79-4c51-97f2-aff1c3cf9ddc. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1353.640373] env[63593]: DEBUG oslo_concurrency.lockutils [req-097b81ef-a0b5-4d8d-9c97-21e76ee36567 req-4f1b95a8-f3a7-47c5-9254-950b924f453e service nova] Acquiring lock "refresh_cache-fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1353.640513] env[63593]: DEBUG oslo_concurrency.lockutils [req-097b81ef-a0b5-4d8d-9c97-21e76ee36567 req-4f1b95a8-f3a7-47c5-9254-950b924f453e service nova] Acquired lock "refresh_cache-fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1353.640668] env[63593]: DEBUG nova.network.neutron [req-097b81ef-a0b5-4d8d-9c97-21e76ee36567 req-4f1b95a8-f3a7-47c5-9254-950b924f453e service nova] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Refreshing network info cache for port 923a100b-6f79-4c51-97f2-aff1c3cf9ddc {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1353.935766] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368044, 'name': CreateVM_Task, 'duration_secs': 0.287848} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1353.935930] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1353.936618] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1353.936782] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1353.937136] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1353.937389] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c6032711-7495-4bae-b232-cb35ab3b84f8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1353.941797] env[63593]: DEBUG oslo_vmware.api [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Waiting for the task: (returnval){ [ 1353.941797] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]520c0e16-d8e0-e7c7-575d-82ebea3e36d0" [ 1353.941797] env[63593]: _type = "Task" [ 1353.941797] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1353.949344] env[63593]: DEBUG oslo_vmware.api [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]520c0e16-d8e0-e7c7-575d-82ebea3e36d0, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1354.402775] env[63593]: DEBUG nova.network.neutron [req-097b81ef-a0b5-4d8d-9c97-21e76ee36567 req-4f1b95a8-f3a7-47c5-9254-950b924f453e service nova] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Updated VIF entry in instance network info cache for port 923a100b-6f79-4c51-97f2-aff1c3cf9ddc. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1354.403229] env[63593]: DEBUG nova.network.neutron [req-097b81ef-a0b5-4d8d-9c97-21e76ee36567 req-4f1b95a8-f3a7-47c5-9254-950b924f453e service nova] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Updating instance_info_cache with network_info: [{"id": "923a100b-6f79-4c51-97f2-aff1c3cf9ddc", "address": "fa:16:3e:62:11:c4", "network": {"id": "6216d3ac-5759-45dd-98ae-f61087681a52", "bridge": "br-int", "label": "tempest-ServersTestJSON-919761004-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "cc0483ca6f804e17a88376fe2ced2ec7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dc6d5964-1106-4345-a26d-185dabd4ff0f", "external-id": "nsx-vlan-transportzone-603", "segmentation_id": 603, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap923a100b-6f", "ovs_interfaceid": "923a100b-6f79-4c51-97f2-aff1c3cf9ddc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1354.451426] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1354.451687] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1354.451911] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1354.906267] env[63593]: DEBUG oslo_concurrency.lockutils [req-097b81ef-a0b5-4d8d-9c97-21e76ee36567 req-4f1b95a8-f3a7-47c5-9254-950b924f453e service nova] Releasing lock "refresh_cache-fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1388.704482] env[63593]: WARNING oslo_vmware.rw_handles [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1388.704482] env[63593]: ERROR oslo_vmware.rw_handles [ 1388.705111] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/c02030e0-b3cc-487d-840e-ace077af2990/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1388.707230] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1388.707546] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Copying Virtual Disk [datastore2] vmware_temp/c02030e0-b3cc-487d-840e-ace077af2990/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/c02030e0-b3cc-487d-840e-ace077af2990/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1388.707762] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-ae46a4b0-1794-4054-ab2b-c158506ca805 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1388.716423] env[63593]: DEBUG oslo_vmware.api [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Waiting for the task: (returnval){ [ 1388.716423] env[63593]: value = "task-1368045" [ 1388.716423] env[63593]: _type = "Task" [ 1388.716423] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1388.724303] env[63593]: DEBUG oslo_vmware.api [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Task: {'id': task-1368045, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1389.226891] env[63593]: DEBUG oslo_vmware.exceptions [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1389.227202] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1389.227744] env[63593]: ERROR nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1389.227744] env[63593]: Faults: ['InvalidArgument'] [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Traceback (most recent call last): [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] yield resources [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] self.driver.spawn(context, instance, image_meta, [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] self._fetch_image_if_missing(context, vi) [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] image_cache(vi, tmp_image_ds_loc) [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] vm_util.copy_virtual_disk( [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] session._wait_for_task(vmdk_copy_task) [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] return self.wait_for_task(task_ref) [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] return evt.wait() [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] result = hub.switch() [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] return self.greenlet.switch() [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] self.f(*self.args, **self.kw) [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] raise exceptions.translate_fault(task_info.error) [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Faults: ['InvalidArgument'] [ 1389.227744] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] [ 1389.228858] env[63593]: INFO nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Terminating instance [ 1389.229572] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1389.229783] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1389.230014] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-9416ff62-a366-45b1-bda8-69380e3fc799 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.232266] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1389.232452] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1389.233165] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0e44a0f-aed6-4a96-9ba3-af92f3496d90 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.239624] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1389.239835] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-c406e0bf-262b-40a2-a98e-26de3743ad10 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.241884] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1389.242074] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1389.243021] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b4d8c6ff-950b-4548-a502-4f50c3d7d6e6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.247673] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for the task: (returnval){ [ 1389.247673] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5297c736-cda0-de20-7b53-61ffb840073f" [ 1389.247673] env[63593]: _type = "Task" [ 1389.247673] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1389.254202] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5297c736-cda0-de20-7b53-61ffb840073f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1389.308434] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1389.308684] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1389.308863] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Deleting the datastore file [datastore2] cf724d13-e34e-4a17-9b54-56190891d5db {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1389.309154] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-00abfbeb-b9c6-4b5c-addb-5703d9d116fd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.314980] env[63593]: DEBUG oslo_vmware.api [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Waiting for the task: (returnval){ [ 1389.314980] env[63593]: value = "task-1368047" [ 1389.314980] env[63593]: _type = "Task" [ 1389.314980] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1389.322152] env[63593]: DEBUG oslo_vmware.api [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Task: {'id': task-1368047, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1389.763175] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1389.763487] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Creating directory with path [datastore2] vmware_temp/1f6eb022-69ef-4a15-9d75-cdaaa72105e9/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1389.763735] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bd649b3e-c388-4579-bb40-88bf1e617509 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.775142] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Created directory with path [datastore2] vmware_temp/1f6eb022-69ef-4a15-9d75-cdaaa72105e9/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1389.775293] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Fetch image to [datastore2] vmware_temp/1f6eb022-69ef-4a15-9d75-cdaaa72105e9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1389.776174] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/1f6eb022-69ef-4a15-9d75-cdaaa72105e9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1389.776314] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da15fe30-6488-4aac-8311-013e674f7a90 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.783053] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c853060-8974-4a64-8de4-7ab6407fe7a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.792316] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05d73647-2952-43d1-b86d-c959a3fd8368 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.825983] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd5b2dfe-d9e3-4d32-911a-0d2f3532450a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.832883] env[63593]: DEBUG oslo_vmware.api [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Task: {'id': task-1368047, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.074355} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1389.834192] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1389.834384] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1389.834553] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1389.834725] env[63593]: INFO nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1389.836791] env[63593]: DEBUG nova.compute.claims [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1389.836958] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1389.837189] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1389.840057] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-71b21509-9b5a-40a2-9ed0-2e19f4f0db10 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.861468] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1389.908051] env[63593]: DEBUG oslo_vmware.rw_handles [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1f6eb022-69ef-4a15-9d75-cdaaa72105e9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1389.968693] env[63593]: DEBUG oslo_vmware.rw_handles [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1389.968894] env[63593]: DEBUG oslo_vmware.rw_handles [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1f6eb022-69ef-4a15-9d75-cdaaa72105e9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1390.511716] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81310a6d-5f5d-430f-a428-e3467cc62942 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1390.518836] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c923001b-d5c9-4913-b693-263abc1feb2f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1390.547756] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9557d584-9b14-465b-bede-adeed80490ee {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1390.554193] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98b600c1-fd5b-4ed2-93c0-42501b4b986f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1390.567424] env[63593]: DEBUG nova.compute.provider_tree [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1391.070242] env[63593]: DEBUG nova.scheduler.client.report [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1391.575055] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.738s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1391.575726] env[63593]: ERROR nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1391.575726] env[63593]: Faults: ['InvalidArgument'] [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Traceback (most recent call last): [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] self.driver.spawn(context, instance, image_meta, [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] self._fetch_image_if_missing(context, vi) [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] image_cache(vi, tmp_image_ds_loc) [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] vm_util.copy_virtual_disk( [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] session._wait_for_task(vmdk_copy_task) [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] return self.wait_for_task(task_ref) [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] return evt.wait() [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] result = hub.switch() [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] return self.greenlet.switch() [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] self.f(*self.args, **self.kw) [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] raise exceptions.translate_fault(task_info.error) [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Faults: ['InvalidArgument'] [ 1391.575726] env[63593]: ERROR nova.compute.manager [instance: cf724d13-e34e-4a17-9b54-56190891d5db] [ 1391.576889] env[63593]: DEBUG nova.compute.utils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1391.578140] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Build of instance cf724d13-e34e-4a17-9b54-56190891d5db was re-scheduled: A specified parameter was not correct: fileType [ 1391.578140] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1391.578554] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1391.578740] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1391.578906] env[63593]: DEBUG nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1391.579080] env[63593]: DEBUG nova.network.neutron [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1392.355418] env[63593]: DEBUG nova.network.neutron [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1392.858421] env[63593]: INFO nova.compute.manager [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Took 1.28 seconds to deallocate network for instance. [ 1393.890678] env[63593]: INFO nova.scheduler.client.report [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Deleted allocations for instance cf724d13-e34e-4a17-9b54-56190891d5db [ 1394.401166] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f986449f-b3bf-4b26-86ca-09c453eebd0a tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "cf724d13-e34e-4a17-9b54-56190891d5db" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 563.361s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1394.401660] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "cf724d13-e34e-4a17-9b54-56190891d5db" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 367.230s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1394.401881] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Acquiring lock "cf724d13-e34e-4a17-9b54-56190891d5db-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1394.402265] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "cf724d13-e34e-4a17-9b54-56190891d5db-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1394.402360] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "cf724d13-e34e-4a17-9b54-56190891d5db-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1394.404216] env[63593]: INFO nova.compute.manager [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Terminating instance [ 1394.405900] env[63593]: DEBUG nova.compute.manager [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1394.406685] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1394.406685] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f53a1674-8cdb-4e0d-810e-3e51dddd7579 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1394.415516] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55c354e0-7991-440b-9374-058c18a7cf9b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1394.441620] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance cf724d13-e34e-4a17-9b54-56190891d5db could not be found. [ 1394.441844] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1394.441989] env[63593]: INFO nova.compute.manager [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1394.442279] env[63593]: DEBUG oslo.service.loopingcall [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1394.442512] env[63593]: DEBUG nova.compute.manager [-] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1394.442607] env[63593]: DEBUG nova.network.neutron [-] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1394.905338] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1394.965069] env[63593]: DEBUG nova.network.neutron [-] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1395.424357] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1395.424615] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1395.426284] env[63593]: INFO nova.compute.claims [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1395.468969] env[63593]: INFO nova.compute.manager [-] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] Took 1.03 seconds to deallocate network for instance. [ 1396.495955] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c65ca599-c250-416a-83b7-80d9497f5bec tempest-ServerActionsTestJSON-106046302 tempest-ServerActionsTestJSON-106046302-project-member] Lock "cf724d13-e34e-4a17-9b54-56190891d5db" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.094s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1396.497344] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "cf724d13-e34e-4a17-9b54-56190891d5db" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 195.456s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1396.497344] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: cf724d13-e34e-4a17-9b54-56190891d5db] During sync_power_state the instance has a pending task (deleting). Skip. [ 1396.497344] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "cf724d13-e34e-4a17-9b54-56190891d5db" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1396.607095] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7c5edb1-d9c4-4581-9608-12cdbd1a378e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1396.615286] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36ba6dc4-da04-429d-a6e1-243e72d1d212 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1396.644198] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-23f00c87-88b6-47a4-8b3f-0f8ac996e48d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1396.651131] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34f33352-b49c-4e04-b46a-a607a6c0757a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1396.664818] env[63593]: DEBUG nova.compute.provider_tree [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1397.167916] env[63593]: DEBUG nova.scheduler.client.report [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1397.672704] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.248s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1397.673264] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1398.178781] env[63593]: DEBUG nova.compute.utils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1398.179470] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1398.179633] env[63593]: DEBUG nova.network.neutron [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1398.216564] env[63593]: DEBUG nova.policy [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a716a2c8efe046e9923bcd6b7956b22a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '60d47be26aac44d198ca43a997485f7c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1398.562285] env[63593]: DEBUG nova.network.neutron [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Successfully created port: c049c09c-92c6-4986-aacd-ace651f2d85f {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1398.685369] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1399.693610] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1399.719059] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1399.719324] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1399.719482] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1399.719659] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1399.719800] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1399.719939] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1399.720188] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1399.720358] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1399.720520] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1399.720674] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1399.720836] env[63593]: DEBUG nova.virt.hardware [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1399.721689] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88026943-05a8-4bfe-93ab-751b12e7333f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1399.729805] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aff03fc9-83ec-4e0b-98ab-94e5d7493d99 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1399.866499] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1399.866686] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1399.866797] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1400.037111] env[63593]: DEBUG nova.compute.manager [req-fa1118ab-a4bc-4470-92b8-9aebe7ff402c req-0ece1653-d028-451b-b2ca-a00255cf61da service nova] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Received event network-vif-plugged-c049c09c-92c6-4986-aacd-ace651f2d85f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1400.037111] env[63593]: DEBUG oslo_concurrency.lockutils [req-fa1118ab-a4bc-4470-92b8-9aebe7ff402c req-0ece1653-d028-451b-b2ca-a00255cf61da service nova] Acquiring lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1400.037111] env[63593]: DEBUG oslo_concurrency.lockutils [req-fa1118ab-a4bc-4470-92b8-9aebe7ff402c req-0ece1653-d028-451b-b2ca-a00255cf61da service nova] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1400.037111] env[63593]: DEBUG oslo_concurrency.lockutils [req-fa1118ab-a4bc-4470-92b8-9aebe7ff402c req-0ece1653-d028-451b-b2ca-a00255cf61da service nova] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1400.037111] env[63593]: DEBUG nova.compute.manager [req-fa1118ab-a4bc-4470-92b8-9aebe7ff402c req-0ece1653-d028-451b-b2ca-a00255cf61da service nova] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] No waiting events found dispatching network-vif-plugged-c049c09c-92c6-4986-aacd-ace651f2d85f {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1400.037111] env[63593]: WARNING nova.compute.manager [req-fa1118ab-a4bc-4470-92b8-9aebe7ff402c req-0ece1653-d028-451b-b2ca-a00255cf61da service nova] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Received unexpected event network-vif-plugged-c049c09c-92c6-4986-aacd-ace651f2d85f for instance with vm_state building and task_state spawning. [ 1400.128210] env[63593]: DEBUG nova.network.neutron [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Successfully updated port: c049c09c-92c6-4986-aacd-ace651f2d85f {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1400.373510] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.373678] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.373808] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.373932] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.374062] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.374205] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.374329] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.374444] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.374561] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.374676] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1400.374793] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1400.374985] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1400.375156] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1400.375339] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1400.375491] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1400.375633] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1400.375773] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1400.375897] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1400.517302] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1400.517548] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1400.631276] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "refresh_cache-2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1400.631457] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired lock "refresh_cache-2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1400.631639] env[63593]: DEBUG nova.network.neutron [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1401.025587] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1401.162720] env[63593]: DEBUG nova.network.neutron [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1401.474355] env[63593]: DEBUG nova.network.neutron [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Updating instance_info_cache with network_info: [{"id": "c049c09c-92c6-4986-aacd-ace651f2d85f", "address": "fa:16:3e:3f:76:65", "network": {"id": "c440ce66-80a9-4360-813a-816782de8ecd", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-114095660-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "60d47be26aac44d198ca43a997485f7c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f77ff7a1-209c-4f3f-b2a0-fd817741e739", "external-id": "nsx-vlan-transportzone-935", "segmentation_id": 935, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc049c09c-92", "ovs_interfaceid": "c049c09c-92c6-4986-aacd-ace651f2d85f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1401.528369] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1401.528596] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1401.528757] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1401.528900] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1401.530252] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de61307c-e4bd-4ee2-afe1-9ba94a914a6b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1401.539238] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1cc1153-fec4-4067-a053-dc1be5eb139a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1401.552950] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61acec0b-3dc2-4fee-b347-d4f48021fa0f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1401.559102] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79e66f81-4b24-4249-812c-2736bee7aac4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1401.587275] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181458MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1401.587425] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1401.587609] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1401.977085] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Releasing lock "refresh_cache-2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1401.977433] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Instance network_info: |[{"id": "c049c09c-92c6-4986-aacd-ace651f2d85f", "address": "fa:16:3e:3f:76:65", "network": {"id": "c440ce66-80a9-4360-813a-816782de8ecd", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-114095660-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "60d47be26aac44d198ca43a997485f7c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f77ff7a1-209c-4f3f-b2a0-fd817741e739", "external-id": "nsx-vlan-transportzone-935", "segmentation_id": 935, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc049c09c-92", "ovs_interfaceid": "c049c09c-92c6-4986-aacd-ace651f2d85f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1401.977856] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:3f:76:65', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f77ff7a1-209c-4f3f-b2a0-fd817741e739', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'c049c09c-92c6-4986-aacd-ace651f2d85f', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1401.985531] env[63593]: DEBUG oslo.service.loopingcall [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1401.985746] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1401.985962] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-ce8fabf2-3718-4183-b263-0283272bae4e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1402.005788] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1402.005788] env[63593]: value = "task-1368048" [ 1402.005788] env[63593]: _type = "Task" [ 1402.005788] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1402.012825] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368048, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1402.061966] env[63593]: DEBUG nova.compute.manager [req-cb5e965a-32c2-43a6-afdc-f40156477bbf req-4e7bb911-9bb7-474d-9d8c-2ebf41f5abad service nova] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Received event network-changed-c049c09c-92c6-4986-aacd-ace651f2d85f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1402.062345] env[63593]: DEBUG nova.compute.manager [req-cb5e965a-32c2-43a6-afdc-f40156477bbf req-4e7bb911-9bb7-474d-9d8c-2ebf41f5abad service nova] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Refreshing instance network info cache due to event network-changed-c049c09c-92c6-4986-aacd-ace651f2d85f. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1402.062609] env[63593]: DEBUG oslo_concurrency.lockutils [req-cb5e965a-32c2-43a6-afdc-f40156477bbf req-4e7bb911-9bb7-474d-9d8c-2ebf41f5abad service nova] Acquiring lock "refresh_cache-2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1402.062753] env[63593]: DEBUG oslo_concurrency.lockutils [req-cb5e965a-32c2-43a6-afdc-f40156477bbf req-4e7bb911-9bb7-474d-9d8c-2ebf41f5abad service nova] Acquired lock "refresh_cache-2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1402.062930] env[63593]: DEBUG nova.network.neutron [req-cb5e965a-32c2-43a6-afdc-f40156477bbf req-4e7bb911-9bb7-474d-9d8c-2ebf41f5abad service nova] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Refreshing network info cache for port c049c09c-92c6-4986-aacd-ace651f2d85f {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1402.516383] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368048, 'name': CreateVM_Task, 'duration_secs': 0.300669} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1402.516614] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1402.517314] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1402.517485] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1402.517813] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1402.518071] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e3deaf5f-b820-4673-99fa-e23c7bb2b069 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1402.522559] env[63593]: DEBUG oslo_vmware.api [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 1402.522559] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]528dc77d-3889-52a3-d42c-4a9f6295d40e" [ 1402.522559] env[63593]: _type = "Task" [ 1402.522559] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1402.530018] env[63593]: DEBUG oslo_vmware.api [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]528dc77d-3889-52a3-d42c-4a9f6295d40e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1402.616405] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.616569] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.616693] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.616809] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.616924] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.617048] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.617164] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.617274] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.617383] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.617490] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1402.938113] env[63593]: DEBUG nova.network.neutron [req-cb5e965a-32c2-43a6-afdc-f40156477bbf req-4e7bb911-9bb7-474d-9d8c-2ebf41f5abad service nova] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Updated VIF entry in instance network info cache for port c049c09c-92c6-4986-aacd-ace651f2d85f. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1402.938475] env[63593]: DEBUG nova.network.neutron [req-cb5e965a-32c2-43a6-afdc-f40156477bbf req-4e7bb911-9bb7-474d-9d8c-2ebf41f5abad service nova] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Updating instance_info_cache with network_info: [{"id": "c049c09c-92c6-4986-aacd-ace651f2d85f", "address": "fa:16:3e:3f:76:65", "network": {"id": "c440ce66-80a9-4360-813a-816782de8ecd", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-114095660-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "60d47be26aac44d198ca43a997485f7c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f77ff7a1-209c-4f3f-b2a0-fd817741e739", "external-id": "nsx-vlan-transportzone-935", "segmentation_id": 935, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapc049c09c-92", "ovs_interfaceid": "c049c09c-92c6-4986-aacd-ace651f2d85f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1403.032613] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1403.032924] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1403.033172] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1403.120015] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance ab5cdb92-e2de-48d9-9abe-9c46b43a89f6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1403.441298] env[63593]: DEBUG oslo_concurrency.lockutils [req-cb5e965a-32c2-43a6-afdc-f40156477bbf req-4e7bb911-9bb7-474d-9d8c-2ebf41f5abad service nova] Releasing lock "refresh_cache-2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1403.624558] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 8d041272-16e4-4e87-95b0-62c5e3c9db2f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1404.128056] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1404.631200] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance e4d93e31-6b0e-4720-bf2a-40794ff79308 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1404.663513] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquiring lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1405.134054] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 28999e3f-02b8-4cb8-adbf-192dc86cb565 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1405.134409] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1405.134409] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1405.303561] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58869a4b-a702-4638-bcb1-919c5e956261 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1405.311048] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3e98f76-f943-4533-b0fc-5e2fddd174fb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1405.341690] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea08b960-f617-4af6-b7c6-7f73de328ccb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1405.350970] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d84ce5f6-de06-439b-9f23-9b7d61ac7802 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1405.361417] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1405.864315] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1406.369869] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1406.369869] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 4.782s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1413.238371] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1420.202703] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1420.202989] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1429.987084] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquiring lock "0484e878-6799-4030-9dcc-d62d32317744" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1429.987376] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "0484e878-6799-4030-9dcc-d62d32317744" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1437.994635] env[63593]: WARNING oslo_vmware.rw_handles [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1437.994635] env[63593]: ERROR oslo_vmware.rw_handles [ 1437.995624] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/1f6eb022-69ef-4a15-9d75-cdaaa72105e9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1437.997147] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1437.997384] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Copying Virtual Disk [datastore2] vmware_temp/1f6eb022-69ef-4a15-9d75-cdaaa72105e9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/1f6eb022-69ef-4a15-9d75-cdaaa72105e9/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1437.997678] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-9c54d15d-ef22-4e17-8441-8c9294660e7f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1438.006427] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for the task: (returnval){ [ 1438.006427] env[63593]: value = "task-1368049" [ 1438.006427] env[63593]: _type = "Task" [ 1438.006427] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1438.014183] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': task-1368049, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1438.519075] env[63593]: DEBUG oslo_vmware.exceptions [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1438.519334] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1438.519901] env[63593]: ERROR nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1438.519901] env[63593]: Faults: ['InvalidArgument'] [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Traceback (most recent call last): [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] yield resources [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] self.driver.spawn(context, instance, image_meta, [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] self._fetch_image_if_missing(context, vi) [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] image_cache(vi, tmp_image_ds_loc) [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] vm_util.copy_virtual_disk( [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] session._wait_for_task(vmdk_copy_task) [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] return self.wait_for_task(task_ref) [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] return evt.wait() [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] result = hub.switch() [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] return self.greenlet.switch() [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] self.f(*self.args, **self.kw) [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] raise exceptions.translate_fault(task_info.error) [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Faults: ['InvalidArgument'] [ 1438.519901] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] [ 1438.520788] env[63593]: INFO nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Terminating instance [ 1438.521758] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1438.521961] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1438.522200] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ada01ab6-aba6-4d7f-ab58-736a89dc19a1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1438.524782] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1438.524971] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1438.525721] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d2569d0-3b5e-4b2b-a72f-0daeceed6504 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1438.532866] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1438.532866] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-3546af92-05c8-4366-ab38-b1341e8a794a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1438.534643] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1438.534822] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1438.535448] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a62ea5cd-15fe-4a18-b466-63a1b5b1b7e4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1438.539944] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for the task: (returnval){ [ 1438.539944] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52a0c9d7-92db-72d9-d10b-4f7466c5972a" [ 1438.539944] env[63593]: _type = "Task" [ 1438.539944] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1438.547030] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52a0c9d7-92db-72d9-d10b-4f7466c5972a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1438.599558] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1438.599775] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1438.599951] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Deleting the datastore file [datastore2] 48e001ae-cc04-4d95-be43-2b2b3556c6db {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1438.600225] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-270c63bb-e35c-4770-a48c-6b2121ba1e2b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1438.606286] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for the task: (returnval){ [ 1438.606286] env[63593]: value = "task-1368051" [ 1438.606286] env[63593]: _type = "Task" [ 1438.606286] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1438.613671] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': task-1368051, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1439.050549] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1439.050838] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Creating directory with path [datastore2] vmware_temp/3ace4288-553e-425a-8595-0191df7ab42f/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1439.051001] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b3d57ef5-decc-4c81-93b0-dc2e08c1ae6b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.062067] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Created directory with path [datastore2] vmware_temp/3ace4288-553e-425a-8595-0191df7ab42f/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1439.062267] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Fetch image to [datastore2] vmware_temp/3ace4288-553e-425a-8595-0191df7ab42f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1439.062434] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/3ace4288-553e-425a-8595-0191df7ab42f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1439.063127] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c944143b-f408-4c0f-9abc-e30adea4c93f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.069338] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd4daa63-ddd9-406f-a78b-16b0358e2632 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.078037] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32bb4403-e31a-470c-9023-23ede4a9166e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.111443] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6fdf4cc-cbc0-4755-8cbb-e59f50fcf0af {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.117965] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': task-1368051, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.077949} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1439.119373] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1439.119600] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1439.119793] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1439.119952] env[63593]: INFO nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1439.121726] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-7d83f8d7-186c-4146-a436-420e771ec694 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.123939] env[63593]: DEBUG nova.compute.claims [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1439.124130] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1439.124342] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1439.145040] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1439.302633] env[63593]: DEBUG oslo_vmware.rw_handles [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/3ace4288-553e-425a-8595-0191df7ab42f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1439.363355] env[63593]: DEBUG oslo_vmware.rw_handles [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1439.363617] env[63593]: DEBUG oslo_vmware.rw_handles [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/3ace4288-553e-425a-8595-0191df7ab42f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1439.794190] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-276d7d81-3351-4af0-a422-f13eb8147d8c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.801927] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a49e1199-fd0d-412b-8f83-767ae3bf2d8c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.830774] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5393013-910f-4aca-97c5-dd1c86e54caa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.837371] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1506d43-b919-4757-98e4-6fa1ca9d0d2b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1439.850751] env[63593]: DEBUG nova.compute.provider_tree [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1440.354539] env[63593]: DEBUG nova.scheduler.client.report [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1440.859763] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.735s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1440.860433] env[63593]: ERROR nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1440.860433] env[63593]: Faults: ['InvalidArgument'] [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Traceback (most recent call last): [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] self.driver.spawn(context, instance, image_meta, [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] self._fetch_image_if_missing(context, vi) [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] image_cache(vi, tmp_image_ds_loc) [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] vm_util.copy_virtual_disk( [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] session._wait_for_task(vmdk_copy_task) [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] return self.wait_for_task(task_ref) [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] return evt.wait() [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] result = hub.switch() [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] return self.greenlet.switch() [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] self.f(*self.args, **self.kw) [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] raise exceptions.translate_fault(task_info.error) [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Faults: ['InvalidArgument'] [ 1440.860433] env[63593]: ERROR nova.compute.manager [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] [ 1440.861296] env[63593]: DEBUG nova.compute.utils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1440.862900] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Build of instance 48e001ae-cc04-4d95-be43-2b2b3556c6db was re-scheduled: A specified parameter was not correct: fileType [ 1440.862900] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1440.863274] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1440.863443] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1440.863611] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1440.863769] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1441.617941] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1442.121078] env[63593]: INFO nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Took 1.26 seconds to deallocate network for instance. [ 1443.158921] env[63593]: INFO nova.scheduler.client.report [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Deleted allocations for instance 48e001ae-cc04-4d95-be43-2b2b3556c6db [ 1443.666939] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 612.028s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1443.668230] env[63593]: DEBUG oslo_concurrency.lockutils [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 416.441s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1443.668451] env[63593]: DEBUG oslo_concurrency.lockutils [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "48e001ae-cc04-4d95-be43-2b2b3556c6db-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1443.668653] env[63593]: DEBUG oslo_concurrency.lockutils [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1443.668815] env[63593]: DEBUG oslo_concurrency.lockutils [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1443.671095] env[63593]: INFO nova.compute.manager [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Terminating instance [ 1443.672731] env[63593]: DEBUG nova.compute.manager [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1443.672929] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1443.673201] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ca79ce5b-571b-4607-8ac4-b5b104740471 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1443.682677] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-310ea30f-0563-4f0a-a96d-3709404aed3a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1443.708109] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 48e001ae-cc04-4d95-be43-2b2b3556c6db could not be found. [ 1443.708300] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1443.708468] env[63593]: INFO nova.compute.manager [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1443.708684] env[63593]: DEBUG oslo.service.loopingcall [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1443.708892] env[63593]: DEBUG nova.compute.manager [-] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1443.708981] env[63593]: DEBUG nova.network.neutron [-] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1444.174093] env[63593]: DEBUG nova.compute.manager [None req-1ef9715f-539f-4eda-b157-38bfee7302ea tempest-ServerPasswordTestJSON-463763678 tempest-ServerPasswordTestJSON-463763678-project-member] [instance: ab5cdb92-e2de-48d9-9abe-9c46b43a89f6] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1444.227027] env[63593]: DEBUG nova.network.neutron [-] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1444.678267] env[63593]: DEBUG nova.compute.manager [None req-1ef9715f-539f-4eda-b157-38bfee7302ea tempest-ServerPasswordTestJSON-463763678 tempest-ServerPasswordTestJSON-463763678-project-member] [instance: ab5cdb92-e2de-48d9-9abe-9c46b43a89f6] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1444.729395] env[63593]: INFO nova.compute.manager [-] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] Took 1.02 seconds to deallocate network for instance. [ 1445.193522] env[63593]: DEBUG oslo_concurrency.lockutils [None req-1ef9715f-539f-4eda-b157-38bfee7302ea tempest-ServerPasswordTestJSON-463763678 tempest-ServerPasswordTestJSON-463763678-project-member] Lock "ab5cdb92-e2de-48d9-9abe-9c46b43a89f6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.457s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1445.695900] env[63593]: DEBUG nova.compute.manager [None req-44d03183-2d52-46c3-9c5d-59c364c1adef tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: 8d041272-16e4-4e87-95b0-62c5e3c9db2f] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1445.751979] env[63593]: DEBUG oslo_concurrency.lockutils [None req-051f0553-aa6c-4513-9713-52bd12cbedd0 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.084s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1445.753452] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 244.712s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1445.753452] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 48e001ae-cc04-4d95-be43-2b2b3556c6db] During sync_power_state the instance has a pending task (deleting). Skip. [ 1445.753609] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "48e001ae-cc04-4d95-be43-2b2b3556c6db" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1446.200403] env[63593]: DEBUG nova.compute.manager [None req-44d03183-2d52-46c3-9c5d-59c364c1adef tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: 8d041272-16e4-4e87-95b0-62c5e3c9db2f] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1446.713265] env[63593]: DEBUG oslo_concurrency.lockutils [None req-44d03183-2d52-46c3-9c5d-59c364c1adef tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "8d041272-16e4-4e87-95b0-62c5e3c9db2f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 223.615s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1447.215620] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1447.738242] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1447.738510] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1447.739964] env[63593]: INFO nova.compute.claims [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1448.889172] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1a2cf50a-710f-4081-bb81-d8c9a9a8514c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1448.896422] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-592e19f3-37d7-40a0-ac86-43c7262061a0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1448.925807] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c18cff00-17b5-486f-84cb-7f298a4f9c89 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1448.932264] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83bdf67c-0a25-4e44-aa50-2e64a9b66fc0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1448.944662] env[63593]: DEBUG nova.compute.provider_tree [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1449.447890] env[63593]: DEBUG nova.scheduler.client.report [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1449.952504] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.214s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1449.953078] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1450.458118] env[63593]: DEBUG nova.compute.utils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1450.459538] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1450.459693] env[63593]: DEBUG nova.network.neutron [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1450.496525] env[63593]: DEBUG nova.policy [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c0a03b9520a349cb99065ce54ec21927', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '87f6487499be40a9b8bf821e1bd4de80', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1450.770225] env[63593]: DEBUG nova.network.neutron [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Successfully created port: f77ac81a-397f-431c-b4f9-575502dd3f1b {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1450.965024] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1451.975371] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1451.999872] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1452.000128] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1452.000382] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1452.000575] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1452.000720] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1452.000864] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1452.001228] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1452.001425] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1452.001600] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1452.001764] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1452.001934] env[63593]: DEBUG nova.virt.hardware [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1452.003039] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3daf1274-70c4-47ff-a2c2-6dd40cd14d03 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1452.012922] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcf035d8-5206-419b-a2a8-4b2cf5dc74c8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1452.179742] env[63593]: DEBUG nova.compute.manager [req-92e1c8ef-0278-4d53-944e-887c27701eab req-5aeb9db5-c259-482e-ba9b-fd0f1f5d95d8 service nova] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Received event network-vif-plugged-f77ac81a-397f-431c-b4f9-575502dd3f1b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1452.179963] env[63593]: DEBUG oslo_concurrency.lockutils [req-92e1c8ef-0278-4d53-944e-887c27701eab req-5aeb9db5-c259-482e-ba9b-fd0f1f5d95d8 service nova] Acquiring lock "a1b46566-4db2-4086-906a-7ba32b9a2813-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1452.180189] env[63593]: DEBUG oslo_concurrency.lockutils [req-92e1c8ef-0278-4d53-944e-887c27701eab req-5aeb9db5-c259-482e-ba9b-fd0f1f5d95d8 service nova] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1452.180355] env[63593]: DEBUG oslo_concurrency.lockutils [req-92e1c8ef-0278-4d53-944e-887c27701eab req-5aeb9db5-c259-482e-ba9b-fd0f1f5d95d8 service nova] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1452.180521] env[63593]: DEBUG nova.compute.manager [req-92e1c8ef-0278-4d53-944e-887c27701eab req-5aeb9db5-c259-482e-ba9b-fd0f1f5d95d8 service nova] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] No waiting events found dispatching network-vif-plugged-f77ac81a-397f-431c-b4f9-575502dd3f1b {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1452.182070] env[63593]: WARNING nova.compute.manager [req-92e1c8ef-0278-4d53-944e-887c27701eab req-5aeb9db5-c259-482e-ba9b-fd0f1f5d95d8 service nova] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Received unexpected event network-vif-plugged-f77ac81a-397f-431c-b4f9-575502dd3f1b for instance with vm_state building and task_state spawning. [ 1452.287524] env[63593]: DEBUG nova.network.neutron [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Successfully updated port: f77ac81a-397f-431c-b4f9-575502dd3f1b {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1452.791022] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "refresh_cache-a1b46566-4db2-4086-906a-7ba32b9a2813" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1452.791186] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "refresh_cache-a1b46566-4db2-4086-906a-7ba32b9a2813" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1452.791393] env[63593]: DEBUG nova.network.neutron [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1453.012822] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1453.013074] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1453.013253] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1453.321249] env[63593]: DEBUG nova.network.neutron [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1453.448988] env[63593]: DEBUG nova.network.neutron [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Updating instance_info_cache with network_info: [{"id": "f77ac81a-397f-431c-b4f9-575502dd3f1b", "address": "fa:16:3e:df:68:5c", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapf77ac81a-39", "ovs_interfaceid": "f77ac81a-397f-431c-b4f9-575502dd3f1b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1453.950841] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "refresh_cache-a1b46566-4db2-4086-906a-7ba32b9a2813" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1453.951191] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Instance network_info: |[{"id": "f77ac81a-397f-431c-b4f9-575502dd3f1b", "address": "fa:16:3e:df:68:5c", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapf77ac81a-39", "ovs_interfaceid": "f77ac81a-397f-431c-b4f9-575502dd3f1b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1453.951629] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:df:68:5c', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a15de394-0367-4921-a5c1-6ac8615e3283', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'f77ac81a-397f-431c-b4f9-575502dd3f1b', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1453.959325] env[63593]: DEBUG oslo.service.loopingcall [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1453.959537] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1453.959758] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-46dcc3ed-8e1b-4a60-b19a-914a6585e3dc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1453.979320] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1453.979320] env[63593]: value = "task-1368052" [ 1453.979320] env[63593]: _type = "Task" [ 1453.979320] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1453.986352] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368052, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1454.213491] env[63593]: DEBUG nova.compute.manager [req-273a16e4-d21d-4f16-90d2-4582b8cc357e req-aaabb22c-a204-40bc-bdf5-d77740f8539c service nova] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Received event network-changed-f77ac81a-397f-431c-b4f9-575502dd3f1b {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1454.213875] env[63593]: DEBUG nova.compute.manager [req-273a16e4-d21d-4f16-90d2-4582b8cc357e req-aaabb22c-a204-40bc-bdf5-d77740f8539c service nova] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Refreshing instance network info cache due to event network-changed-f77ac81a-397f-431c-b4f9-575502dd3f1b. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1454.214160] env[63593]: DEBUG oslo_concurrency.lockutils [req-273a16e4-d21d-4f16-90d2-4582b8cc357e req-aaabb22c-a204-40bc-bdf5-d77740f8539c service nova] Acquiring lock "refresh_cache-a1b46566-4db2-4086-906a-7ba32b9a2813" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1454.214348] env[63593]: DEBUG oslo_concurrency.lockutils [req-273a16e4-d21d-4f16-90d2-4582b8cc357e req-aaabb22c-a204-40bc-bdf5-d77740f8539c service nova] Acquired lock "refresh_cache-a1b46566-4db2-4086-906a-7ba32b9a2813" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1454.214542] env[63593]: DEBUG nova.network.neutron [req-273a16e4-d21d-4f16-90d2-4582b8cc357e req-aaabb22c-a204-40bc-bdf5-d77740f8539c service nova] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Refreshing network info cache for port f77ac81a-397f-431c-b4f9-575502dd3f1b {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1454.489860] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368052, 'name': CreateVM_Task, 'duration_secs': 0.287768} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1454.489860] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1454.490412] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1454.490576] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1454.490890] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1454.491148] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-76e6beb1-a0f8-41d1-ace8-d93dce52fd06 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1454.495597] env[63593]: DEBUG oslo_vmware.api [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 1454.495597] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]520b99cc-6dd1-b56c-0fc8-a8785253f523" [ 1454.495597] env[63593]: _type = "Task" [ 1454.495597] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1454.503131] env[63593]: DEBUG oslo_vmware.api [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]520b99cc-6dd1-b56c-0fc8-a8785253f523, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1454.514652] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1454.514885] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11226}} [ 1454.893523] env[63593]: DEBUG nova.network.neutron [req-273a16e4-d21d-4f16-90d2-4582b8cc357e req-aaabb22c-a204-40bc-bdf5-d77740f8539c service nova] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Updated VIF entry in instance network info cache for port f77ac81a-397f-431c-b4f9-575502dd3f1b. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1454.893859] env[63593]: DEBUG nova.network.neutron [req-273a16e4-d21d-4f16-90d2-4582b8cc357e req-aaabb22c-a204-40bc-bdf5-d77740f8539c service nova] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Updating instance_info_cache with network_info: [{"id": "f77ac81a-397f-431c-b4f9-575502dd3f1b", "address": "fa:16:3e:df:68:5c", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapf77ac81a-39", "ovs_interfaceid": "f77ac81a-397f-431c-b4f9-575502dd3f1b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1455.006424] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1455.006638] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1455.006838] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1455.017366] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] There are 0 instances to clean {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11235}} [ 1455.396994] env[63593]: DEBUG oslo_concurrency.lockutils [req-273a16e4-d21d-4f16-90d2-4582b8cc357e req-aaabb22c-a204-40bc-bdf5-d77740f8539c service nova] Releasing lock "refresh_cache-a1b46566-4db2-4086-906a-7ba32b9a2813" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1455.515708] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1455.515965] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1455.516081] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1456.021982] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.021982] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.021982] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.021982] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.021982] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.021982] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.021982] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.021982] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.022307] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.022307] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1456.022307] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1456.022476] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1457.012967] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1457.013238] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1457.456622] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "a1b46566-4db2-4086-906a-7ba32b9a2813" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1460.012850] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1461.012582] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1461.012847] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1461.013015] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances with incomplete migration {{(pid=63593) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11264}} [ 1462.511770] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1463.012724] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1463.520140] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1463.520140] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1463.520140] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1463.520140] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1463.520140] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9804edb7-8c53-431c-9817-47ed91f634b3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1463.526603] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8797c28-c201-46e0-845b-fdae039cf7db {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1463.540306] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad2ae448-8da6-404e-9dfc-2a373bf26ec8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1463.546469] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9db4f15-e921-4cd4-b3ae-4737658f748f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1463.576243] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181456MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1463.576521] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1463.576618] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1464.666035] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1464.666035] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1464.666035] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1464.666383] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1464.666383] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1464.666383] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1464.666383] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1464.666501] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1464.666607] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1464.666715] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1465.169822] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance e4d93e31-6b0e-4720-bf2a-40794ff79308 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1465.674259] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 28999e3f-02b8-4cb8-adbf-192dc86cb565 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1466.178418] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1466.680838] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1466.681147] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1466.681292] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1466.698637] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing inventories for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 1466.712813] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Updating ProviderTree inventory for provider 7345f17f-3af2-4b0a-9521-0292dc691877 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 1466.713009] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Updating inventory in ProviderTree for provider 7345f17f-3af2-4b0a-9521-0292dc691877 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1466.724399] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing aggregate associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, aggregates: None {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 1466.742347] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing trait associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 1466.887801] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92e31d4a-a394-49c4-9cc0-6983fb781c24 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1466.897476] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc5d1694-55cf-4c5f-a171-583796835d2a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1466.933841] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9d708fc-7999-40c4-86e6-f420796fee30 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1466.940135] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95c6db6e-cf1b-418a-a8ed-ae21b10f8f55 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1466.952753] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1467.456078] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1467.960858] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1467.961199] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 4.384s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1468.714711] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1468.714992] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1488.808073] env[63593]: WARNING oslo_vmware.rw_handles [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1488.808073] env[63593]: ERROR oslo_vmware.rw_handles [ 1488.808667] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/3ace4288-553e-425a-8595-0191df7ab42f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1488.810847] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1488.811120] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Copying Virtual Disk [datastore2] vmware_temp/3ace4288-553e-425a-8595-0191df7ab42f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/3ace4288-553e-425a-8595-0191df7ab42f/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1488.811424] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-469ce193-24a6-4f09-a663-67761bd876b4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1488.819667] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for the task: (returnval){ [ 1488.819667] env[63593]: value = "task-1368053" [ 1488.819667] env[63593]: _type = "Task" [ 1488.819667] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1488.827642] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': task-1368053, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1489.330675] env[63593]: DEBUG oslo_vmware.exceptions [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1489.330983] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1489.331549] env[63593]: ERROR nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1489.331549] env[63593]: Faults: ['InvalidArgument'] [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Traceback (most recent call last): [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] yield resources [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] self.driver.spawn(context, instance, image_meta, [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] self._fetch_image_if_missing(context, vi) [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] image_cache(vi, tmp_image_ds_loc) [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] vm_util.copy_virtual_disk( [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] session._wait_for_task(vmdk_copy_task) [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] return self.wait_for_task(task_ref) [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] return evt.wait() [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] result = hub.switch() [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] return self.greenlet.switch() [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] self.f(*self.args, **self.kw) [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] raise exceptions.translate_fault(task_info.error) [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Faults: ['InvalidArgument'] [ 1489.331549] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] [ 1489.332518] env[63593]: INFO nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Terminating instance [ 1489.333477] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1489.333678] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1489.334304] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1489.334492] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1489.334711] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bb74ec89-d025-4a48-8487-dfbf4482aa46 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.336938] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db240a81-0acd-42da-921b-bd99ebc67a1b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.343964] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1489.344183] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-592e86c0-5d2b-45d9-a4db-ee49e46211a4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.346330] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1489.346521] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1489.347593] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-28be3930-09e1-4c43-bc7f-4b28e4746b1b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.351813] env[63593]: DEBUG oslo_vmware.api [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 1489.351813] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5227a57b-7f2f-03a4-141a-5d86f0423586" [ 1489.351813] env[63593]: _type = "Task" [ 1489.351813] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1489.358916] env[63593]: DEBUG oslo_vmware.api [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5227a57b-7f2f-03a4-141a-5d86f0423586, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1489.417490] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1489.417767] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1489.417914] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Deleting the datastore file [datastore2] 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1489.418205] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-520d4b6c-31db-47d3-bc14-d79599ec2c36 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.424629] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for the task: (returnval){ [ 1489.424629] env[63593]: value = "task-1368055" [ 1489.424629] env[63593]: _type = "Task" [ 1489.424629] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1489.433486] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': task-1368055, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1489.861586] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1489.861957] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] vmware_temp/a2000514-5bc9-48d5-8133-32806ebd7570/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1489.862085] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-948ef63f-c189-4070-9d31-be38b02fbf05 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.873203] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] vmware_temp/a2000514-5bc9-48d5-8133-32806ebd7570/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1489.873404] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Fetch image to [datastore2] vmware_temp/a2000514-5bc9-48d5-8133-32806ebd7570/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1489.873571] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/a2000514-5bc9-48d5-8133-32806ebd7570/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1489.874276] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f94bd8e-812e-464a-b4c6-109d09837215 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.880494] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10a62277-0c0f-455f-af46-d8346056fa9c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.889069] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebef5682-66e7-4a20-a32c-0369c0d42049 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.918187] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe697d5a-871d-4dda-9c06-004c80737f94 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.923621] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-3ce5eba3-de7a-4be2-af40-affbd497288c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1489.932367] env[63593]: DEBUG oslo_vmware.api [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Task: {'id': task-1368055, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.076879} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1489.932595] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1489.932770] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1489.932935] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1489.933133] env[63593]: INFO nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1489.935269] env[63593]: DEBUG nova.compute.claims [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1489.935455] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1489.935665] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1489.946711] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1489.996380] env[63593]: DEBUG oslo_vmware.rw_handles [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/a2000514-5bc9-48d5-8133-32806ebd7570/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1490.056997] env[63593]: DEBUG oslo_vmware.rw_handles [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1490.057206] env[63593]: DEBUG oslo_vmware.rw_handles [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/a2000514-5bc9-48d5-8133-32806ebd7570/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1490.587473] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27b4dc0b-eeb0-4242-8c1a-29956cccad88 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1490.594759] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a077c50c-37fe-4ccd-9a20-ac316c50f440 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1490.623919] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e1b8891-6531-47c1-a1f2-847da20b2e5e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1490.630709] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d665399f-0757-43d6-a3da-250466022d71 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1490.643033] env[63593]: DEBUG nova.compute.provider_tree [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1491.146364] env[63593]: DEBUG nova.scheduler.client.report [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1491.651175] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.715s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1491.651746] env[63593]: ERROR nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1491.651746] env[63593]: Faults: ['InvalidArgument'] [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Traceback (most recent call last): [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] self.driver.spawn(context, instance, image_meta, [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] self._fetch_image_if_missing(context, vi) [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] image_cache(vi, tmp_image_ds_loc) [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] vm_util.copy_virtual_disk( [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] session._wait_for_task(vmdk_copy_task) [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] return self.wait_for_task(task_ref) [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] return evt.wait() [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] result = hub.switch() [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] return self.greenlet.switch() [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] self.f(*self.args, **self.kw) [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] raise exceptions.translate_fault(task_info.error) [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Faults: ['InvalidArgument'] [ 1491.651746] env[63593]: ERROR nova.compute.manager [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] [ 1491.652578] env[63593]: DEBUG nova.compute.utils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1491.654306] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Build of instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 was re-scheduled: A specified parameter was not correct: fileType [ 1491.654306] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1491.654663] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1491.654832] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1491.655006] env[63593]: DEBUG nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1491.655171] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1492.379852] env[63593]: DEBUG nova.network.neutron [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1492.883328] env[63593]: INFO nova.compute.manager [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Took 1.23 seconds to deallocate network for instance. [ 1493.917405] env[63593]: INFO nova.scheduler.client.report [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Deleted allocations for instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 [ 1494.425382] env[63593]: DEBUG oslo_concurrency.lockutils [None req-de0e39fa-5886-4272-a046-af750e38ba48 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 662.752s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1494.426698] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 467.263s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1494.426922] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Acquiring lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1494.427140] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1494.427307] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1494.429164] env[63593]: INFO nova.compute.manager [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Terminating instance [ 1494.431059] env[63593]: DEBUG nova.compute.manager [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1494.431251] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1494.431517] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a755e4bc-8f10-444d-bab5-3c37ac34a9e7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1494.440286] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad33a6d2-5ecf-47ad-948c-c3b0aa5817a0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1494.467183] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6a21b30c-1e56-4c0c-ad7d-e0987f70d203 could not be found. [ 1494.467514] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1494.467764] env[63593]: INFO nova.compute.manager [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1494.468034] env[63593]: DEBUG oslo.service.loopingcall [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1494.468284] env[63593]: DEBUG nova.compute.manager [-] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1494.468380] env[63593]: DEBUG nova.network.neutron [-] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1494.930584] env[63593]: DEBUG nova.compute.manager [None req-cefb2407-ed33-43ef-8b98-dfcc34f61a05 tempest-AttachInterfacesUnderV243Test-729252606 tempest-AttachInterfacesUnderV243Test-729252606-project-member] [instance: e4d93e31-6b0e-4720-bf2a-40794ff79308] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1494.987048] env[63593]: DEBUG nova.network.neutron [-] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1495.434418] env[63593]: DEBUG nova.compute.manager [None req-cefb2407-ed33-43ef-8b98-dfcc34f61a05 tempest-AttachInterfacesUnderV243Test-729252606 tempest-AttachInterfacesUnderV243Test-729252606-project-member] [instance: e4d93e31-6b0e-4720-bf2a-40794ff79308] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1495.489041] env[63593]: INFO nova.compute.manager [-] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] Took 1.02 seconds to deallocate network for instance. [ 1495.949492] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cefb2407-ed33-43ef-8b98-dfcc34f61a05 tempest-AttachInterfacesUnderV243Test-729252606 tempest-AttachInterfacesUnderV243Test-729252606-project-member] Lock "e4d93e31-6b0e-4720-bf2a-40794ff79308" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 211.878s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1496.452427] env[63593]: DEBUG nova.compute.manager [None req-a79d8d6c-928d-4efb-ba68-a2cf1949de57 tempest-ServersAaction247Test-1718206514 tempest-ServersAaction247Test-1718206514-project-member] [instance: 28999e3f-02b8-4cb8-adbf-192dc86cb565] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1496.512484] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b4054995-9413-498b-b2a7-04284acbd9c4 tempest-ListServersNegativeTestJSON-367705171 tempest-ListServersNegativeTestJSON-367705171-project-member] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.086s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1496.513387] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 295.472s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1496.513583] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 6a21b30c-1e56-4c0c-ad7d-e0987f70d203] During sync_power_state the instance has a pending task (deleting). Skip. [ 1496.513756] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "6a21b30c-1e56-4c0c-ad7d-e0987f70d203" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1496.956709] env[63593]: DEBUG nova.compute.manager [None req-a79d8d6c-928d-4efb-ba68-a2cf1949de57 tempest-ServersAaction247Test-1718206514 tempest-ServersAaction247Test-1718206514-project-member] [instance: 28999e3f-02b8-4cb8-adbf-192dc86cb565] Instance disappeared before build. {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 1497.469231] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a79d8d6c-928d-4efb-ba68-a2cf1949de57 tempest-ServersAaction247Test-1718206514 tempest-ServersAaction247Test-1718206514-project-member] Lock "28999e3f-02b8-4cb8-adbf-192dc86cb565" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 211.319s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1497.972532] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1498.493462] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1498.493716] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1498.496206] env[63593]: INFO nova.compute.claims [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1499.630964] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70636064-3262-4879-894d-a25b315eecd2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1499.638452] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c494691-8138-4ff9-8257-6842b0ca00f6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1499.667420] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-298d24df-07e0-4bfc-b5e6-46df4c2e8402 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1499.674076] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e72f80da-d8db-4662-9c77-2826087e32a4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1499.686583] env[63593]: DEBUG nova.compute.provider_tree [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1500.189773] env[63593]: DEBUG nova.scheduler.client.report [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1500.694973] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.201s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1500.695514] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1501.200687] env[63593]: DEBUG nova.compute.utils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1501.202221] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1501.202395] env[63593]: DEBUG nova.network.neutron [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1501.241411] env[63593]: DEBUG nova.policy [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd5f23913d4a1411282f081fb5ed46a11', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '220c072768c1441fbfe88f849cd6eb03', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1501.483396] env[63593]: DEBUG nova.network.neutron [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Successfully created port: 8ca28c46-bbd9-4f66-acae-43673ed7d179 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1501.705547] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1502.714303] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1502.739177] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1502.739424] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1502.739579] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1502.739788] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1502.739934] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1502.740089] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1502.740294] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1502.740446] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1502.740608] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1502.740763] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1502.740927] env[63593]: DEBUG nova.virt.hardware [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1502.741801] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9ba171c-9ae2-4306-b4ae-15b8c4e6e745 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1502.749681] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d58f25fa-5232-4e54-8ef6-5ae7bd925c0c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1502.863770] env[63593]: DEBUG nova.compute.manager [req-f9666683-9764-47c5-9cb2-50817288ddd7 req-2935c577-9b16-4d81-8f74-560bca349781 service nova] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Received event network-vif-plugged-8ca28c46-bbd9-4f66-acae-43673ed7d179 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1502.863996] env[63593]: DEBUG oslo_concurrency.lockutils [req-f9666683-9764-47c5-9cb2-50817288ddd7 req-2935c577-9b16-4d81-8f74-560bca349781 service nova] Acquiring lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1502.864225] env[63593]: DEBUG oslo_concurrency.lockutils [req-f9666683-9764-47c5-9cb2-50817288ddd7 req-2935c577-9b16-4d81-8f74-560bca349781 service nova] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1502.864392] env[63593]: DEBUG oslo_concurrency.lockutils [req-f9666683-9764-47c5-9cb2-50817288ddd7 req-2935c577-9b16-4d81-8f74-560bca349781 service nova] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1502.864554] env[63593]: DEBUG nova.compute.manager [req-f9666683-9764-47c5-9cb2-50817288ddd7 req-2935c577-9b16-4d81-8f74-560bca349781 service nova] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] No waiting events found dispatching network-vif-plugged-8ca28c46-bbd9-4f66-acae-43673ed7d179 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1502.864714] env[63593]: WARNING nova.compute.manager [req-f9666683-9764-47c5-9cb2-50817288ddd7 req-2935c577-9b16-4d81-8f74-560bca349781 service nova] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Received unexpected event network-vif-plugged-8ca28c46-bbd9-4f66-acae-43673ed7d179 for instance with vm_state building and task_state spawning. [ 1502.948998] env[63593]: DEBUG nova.network.neutron [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Successfully updated port: 8ca28c46-bbd9-4f66-acae-43673ed7d179 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1503.451355] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "refresh_cache-abaeda1e-4ad8-4553-9fb0-f94e2cd98059" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1503.451517] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "refresh_cache-abaeda1e-4ad8-4553-9fb0-f94e2cd98059" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1503.451669] env[63593]: DEBUG nova.network.neutron [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1504.006321] env[63593]: DEBUG nova.network.neutron [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1504.190098] env[63593]: DEBUG nova.network.neutron [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Updating instance_info_cache with network_info: [{"id": "8ca28c46-bbd9-4f66-acae-43673ed7d179", "address": "fa:16:3e:4c:55:d4", "network": {"id": "fb445387-b1c5-45bc-a832-c443e70d8604", "bridge": "br-int", "label": "tempest-ServerDiskConfigTestJSON-429488907-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "220c072768c1441fbfe88f849cd6eb03", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bafe8721-91d4-4127-b215-d9e8e27947dc", "external-id": "nsx-vlan-transportzone-680", "segmentation_id": 680, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8ca28c46-bb", "ovs_interfaceid": "8ca28c46-bbd9-4f66-acae-43673ed7d179", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1504.692275] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "refresh_cache-abaeda1e-4ad8-4553-9fb0-f94e2cd98059" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1504.692616] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Instance network_info: |[{"id": "8ca28c46-bbd9-4f66-acae-43673ed7d179", "address": "fa:16:3e:4c:55:d4", "network": {"id": "fb445387-b1c5-45bc-a832-c443e70d8604", "bridge": "br-int", "label": "tempest-ServerDiskConfigTestJSON-429488907-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "220c072768c1441fbfe88f849cd6eb03", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bafe8721-91d4-4127-b215-d9e8e27947dc", "external-id": "nsx-vlan-transportzone-680", "segmentation_id": 680, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8ca28c46-bb", "ovs_interfaceid": "8ca28c46-bbd9-4f66-acae-43673ed7d179", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1504.693047] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:4c:55:d4', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'bafe8721-91d4-4127-b215-d9e8e27947dc', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '8ca28c46-bbd9-4f66-acae-43673ed7d179', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1504.700462] env[63593]: DEBUG oslo.service.loopingcall [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1504.700670] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1504.700882] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-8764bc94-ca0c-4ccd-9884-b576b0c0e71b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1504.721906] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1504.721906] env[63593]: value = "task-1368056" [ 1504.721906] env[63593]: _type = "Task" [ 1504.721906] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1504.732778] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368056, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1504.893388] env[63593]: DEBUG nova.compute.manager [req-f0172c64-1379-4f81-8b41-62de10387bff req-46deca63-93f8-42c4-a628-3327624542b7 service nova] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Received event network-changed-8ca28c46-bbd9-4f66-acae-43673ed7d179 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1504.893546] env[63593]: DEBUG nova.compute.manager [req-f0172c64-1379-4f81-8b41-62de10387bff req-46deca63-93f8-42c4-a628-3327624542b7 service nova] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Refreshing instance network info cache due to event network-changed-8ca28c46-bbd9-4f66-acae-43673ed7d179. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1504.893737] env[63593]: DEBUG oslo_concurrency.lockutils [req-f0172c64-1379-4f81-8b41-62de10387bff req-46deca63-93f8-42c4-a628-3327624542b7 service nova] Acquiring lock "refresh_cache-abaeda1e-4ad8-4553-9fb0-f94e2cd98059" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1504.893832] env[63593]: DEBUG oslo_concurrency.lockutils [req-f0172c64-1379-4f81-8b41-62de10387bff req-46deca63-93f8-42c4-a628-3327624542b7 service nova] Acquired lock "refresh_cache-abaeda1e-4ad8-4553-9fb0-f94e2cd98059" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1504.893985] env[63593]: DEBUG nova.network.neutron [req-f0172c64-1379-4f81-8b41-62de10387bff req-46deca63-93f8-42c4-a628-3327624542b7 service nova] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Refreshing network info cache for port 8ca28c46-bbd9-4f66-acae-43673ed7d179 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1505.232427] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368056, 'name': CreateVM_Task, 'duration_secs': 0.299817} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1505.232664] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1505.239752] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1505.240590] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1505.240590] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1505.240590] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-35ce0c75-7bc7-4dad-8e32-2b5e8ca87396 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1505.244907] env[63593]: DEBUG oslo_vmware.api [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for the task: (returnval){ [ 1505.244907] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]521cff26-e873-75d7-3d52-5fbcc29509b5" [ 1505.244907] env[63593]: _type = "Task" [ 1505.244907] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1505.252432] env[63593]: DEBUG oslo_vmware.api [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]521cff26-e873-75d7-3d52-5fbcc29509b5, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1505.576443] env[63593]: DEBUG nova.network.neutron [req-f0172c64-1379-4f81-8b41-62de10387bff req-46deca63-93f8-42c4-a628-3327624542b7 service nova] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Updated VIF entry in instance network info cache for port 8ca28c46-bbd9-4f66-acae-43673ed7d179. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1505.576821] env[63593]: DEBUG nova.network.neutron [req-f0172c64-1379-4f81-8b41-62de10387bff req-46deca63-93f8-42c4-a628-3327624542b7 service nova] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Updating instance_info_cache with network_info: [{"id": "8ca28c46-bbd9-4f66-acae-43673ed7d179", "address": "fa:16:3e:4c:55:d4", "network": {"id": "fb445387-b1c5-45bc-a832-c443e70d8604", "bridge": "br-int", "label": "tempest-ServerDiskConfigTestJSON-429488907-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "220c072768c1441fbfe88f849cd6eb03", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "bafe8721-91d4-4127-b215-d9e8e27947dc", "external-id": "nsx-vlan-transportzone-680", "segmentation_id": 680, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8ca28c46-bb", "ovs_interfaceid": "8ca28c46-bbd9-4f66-acae-43673ed7d179", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1505.756749] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1505.757110] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1505.757442] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1506.079406] env[63593]: DEBUG oslo_concurrency.lockutils [req-f0172c64-1379-4f81-8b41-62de10387bff req-46deca63-93f8-42c4-a628-3327624542b7 service nova] Releasing lock "refresh_cache-abaeda1e-4ad8-4553-9fb0-f94e2cd98059" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1518.961526] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1518.961803] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1518.961836] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1519.466926] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.467194] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.467270] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.467348] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.467500] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.467629] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.467746] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.467862] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.467977] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.468105] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1519.468223] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1519.468413] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1519.468565] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1519.468738] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1519.468923] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1519.469063] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1520.012511] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1522.012969] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1524.008498] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1524.012196] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1524.516808] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1524.517082] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1524.517258] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1524.517419] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1524.518365] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1548faca-c385-4f86-a0f6-ea0a4e5d544d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1524.526149] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-611af887-846a-485d-acec-afd4b6ce0038 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1524.539332] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfdcaf09-6db3-4e01-8bc1-8748ce10f376 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1524.545433] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2afadf4d-cfd2-405f-9a44-f998effec223 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1524.574077] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181413MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1524.574222] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1524.574410] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1525.605808] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1525.606075] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1525.606114] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1525.606207] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1525.606328] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1525.606442] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1525.606552] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1525.606672] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1525.606808] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1525.606923] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1526.109895] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1526.613499] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1526.613831] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1526.613889] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1526.740102] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d380909-4695-4a8f-9477-1a4fb4f966ca {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1526.747323] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea125fc7-5ad6-4031-8aa5-82ad9631b01b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1526.775491] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bd29df6-d82a-4f32-b20b-40c04c42419b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1526.782118] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dc25df6-185c-44aa-9ae8-91c25cc8fbcf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1526.795179] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1527.298205] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1527.802904] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1527.803250] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.229s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1528.799322] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1537.670033] env[63593]: WARNING oslo_vmware.rw_handles [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1537.670033] env[63593]: ERROR oslo_vmware.rw_handles [ 1537.670033] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/a2000514-5bc9-48d5-8133-32806ebd7570/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1537.673037] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1537.673037] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Copying Virtual Disk [datastore2] vmware_temp/a2000514-5bc9-48d5-8133-32806ebd7570/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/a2000514-5bc9-48d5-8133-32806ebd7570/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1537.673037] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-0a632e7c-9795-4cc7-b2fe-e3a6ada0dfdf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1537.680963] env[63593]: DEBUG oslo_vmware.api [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 1537.680963] env[63593]: value = "task-1368057" [ 1537.680963] env[63593]: _type = "Task" [ 1537.680963] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1537.688766] env[63593]: DEBUG oslo_vmware.api [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368057, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1538.191380] env[63593]: DEBUG oslo_vmware.exceptions [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1538.191605] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1538.192194] env[63593]: ERROR nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1538.192194] env[63593]: Faults: ['InvalidArgument'] [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Traceback (most recent call last): [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] yield resources [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] self.driver.spawn(context, instance, image_meta, [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] self._fetch_image_if_missing(context, vi) [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] image_cache(vi, tmp_image_ds_loc) [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] vm_util.copy_virtual_disk( [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] session._wait_for_task(vmdk_copy_task) [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] return self.wait_for_task(task_ref) [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] return evt.wait() [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] result = hub.switch() [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] return self.greenlet.switch() [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] self.f(*self.args, **self.kw) [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] raise exceptions.translate_fault(task_info.error) [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Faults: ['InvalidArgument'] [ 1538.192194] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] [ 1538.193232] env[63593]: INFO nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Terminating instance [ 1538.194007] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1538.194256] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1538.194436] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-20d2ad5f-aa45-4b21-9896-93ab47cae700 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.196500] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1538.196684] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1538.197383] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d49ca829-d613-4196-8a90-cb6fb35b2302 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.205131] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1538.206159] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-ab24aceb-d005-4899-9a38-b8d275405387 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.207553] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1538.207739] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1538.208813] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-9dbd2d5e-cc84-4966-bf52-26151cb84fd7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.214032] env[63593]: DEBUG oslo_vmware.api [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Waiting for the task: (returnval){ [ 1538.214032] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52fc897e-9808-f210-0268-9a34adf216d8" [ 1538.214032] env[63593]: _type = "Task" [ 1538.214032] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1538.229145] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1538.229381] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Creating directory with path [datastore2] vmware_temp/4577a9ba-f25b-4cd7-9603-71e6d4b34f67/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1538.229592] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8cb07574-ae03-4d80-8eee-7a13b70eec55 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.241106] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Created directory with path [datastore2] vmware_temp/4577a9ba-f25b-4cd7-9603-71e6d4b34f67/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1538.241347] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Fetch image to [datastore2] vmware_temp/4577a9ba-f25b-4cd7-9603-71e6d4b34f67/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1538.241551] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/4577a9ba-f25b-4cd7-9603-71e6d4b34f67/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1538.242275] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4aead04c-9031-46b9-8161-29f8fda0e6e5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.249629] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e2f6776-99c3-45e8-ad29-106be1c06d73 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.258745] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3d7ed3c-1779-4103-a4c9-7736b746ef7a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.294289] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d87b4a5b-f334-462c-9dc6-17e4bff25f57 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.296877] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1538.297102] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1538.297279] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleting the datastore file [datastore2] 4e8f8346-70b0-4ae4-925c-50094ac5a07a {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1538.297512] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-c76d7a0b-efa0-4820-84c6-17f4330d8ae8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.302073] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4233655d-9d04-4f5f-b012-e76570197d3f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1538.304642] env[63593]: DEBUG oslo_vmware.api [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 1538.304642] env[63593]: value = "task-1368059" [ 1538.304642] env[63593]: _type = "Task" [ 1538.304642] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1538.313022] env[63593]: DEBUG oslo_vmware.api [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368059, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1538.321973] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1538.369067] env[63593]: DEBUG oslo_vmware.rw_handles [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4577a9ba-f25b-4cd7-9603-71e6d4b34f67/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1538.432179] env[63593]: DEBUG oslo_vmware.rw_handles [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1538.432665] env[63593]: DEBUG oslo_vmware.rw_handles [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4577a9ba-f25b-4cd7-9603-71e6d4b34f67/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1538.814668] env[63593]: DEBUG oslo_vmware.api [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368059, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.071876} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1538.814915] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1538.815097] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1538.815268] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1538.815439] env[63593]: INFO nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1538.817503] env[63593]: DEBUG nova.compute.claims [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1538.817672] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1538.817884] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1539.461636] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28512cc1-accf-4ea1-b9ea-a3a661de713e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1539.469315] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3596931c-6b84-4685-8830-1d7a115c5936 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1539.498832] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa83cf13-965f-42ca-b1df-204878320fd3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1539.505552] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a421c1e-32dc-4a66-b31a-4728e8e8a36a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1539.517985] env[63593]: DEBUG nova.compute.provider_tree [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1540.021079] env[63593]: DEBUG nova.scheduler.client.report [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1540.526612] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.708s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1540.527265] env[63593]: ERROR nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1540.527265] env[63593]: Faults: ['InvalidArgument'] [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Traceback (most recent call last): [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] self.driver.spawn(context, instance, image_meta, [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] self._fetch_image_if_missing(context, vi) [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] image_cache(vi, tmp_image_ds_loc) [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] vm_util.copy_virtual_disk( [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] session._wait_for_task(vmdk_copy_task) [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] return self.wait_for_task(task_ref) [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] return evt.wait() [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] result = hub.switch() [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] return self.greenlet.switch() [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] self.f(*self.args, **self.kw) [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] raise exceptions.translate_fault(task_info.error) [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Faults: ['InvalidArgument'] [ 1540.527265] env[63593]: ERROR nova.compute.manager [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] [ 1540.528240] env[63593]: DEBUG nova.compute.utils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1540.529928] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Build of instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a was re-scheduled: A specified parameter was not correct: fileType [ 1540.529928] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1540.530320] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1540.530506] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1540.530678] env[63593]: DEBUG nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1540.530836] env[63593]: DEBUG nova.network.neutron [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1541.268725] env[63593]: DEBUG nova.network.neutron [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1541.771298] env[63593]: INFO nova.compute.manager [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Took 1.24 seconds to deallocate network for instance. [ 1542.802316] env[63593]: INFO nova.scheduler.client.report [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted allocations for instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a [ 1543.311678] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fb5d4edc-c0ae-4922-8691-5e616b067d61 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 674.615s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1543.312458] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 478.582s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1543.312678] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1543.312877] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1543.313051] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1543.315014] env[63593]: INFO nova.compute.manager [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Terminating instance [ 1543.316563] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "refresh_cache-4e8f8346-70b0-4ae4-925c-50094ac5a07a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1543.316711] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "refresh_cache-4e8f8346-70b0-4ae4-925c-50094ac5a07a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1543.316901] env[63593]: DEBUG nova.network.neutron [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1543.816317] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1543.838153] env[63593]: DEBUG nova.network.neutron [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1543.913820] env[63593]: DEBUG nova.network.neutron [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1544.340676] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1544.340934] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1544.342397] env[63593]: INFO nova.compute.claims [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1544.416424] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "refresh_cache-4e8f8346-70b0-4ae4-925c-50094ac5a07a" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1544.416853] env[63593]: DEBUG nova.compute.manager [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1544.417095] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1544.417410] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5d61d5e5-3394-4762-92ef-609018d0ea0f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1544.426107] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a57502ce-bb16-485f-8a94-5ae584a90a9e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1544.451747] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4e8f8346-70b0-4ae4-925c-50094ac5a07a could not be found. [ 1544.451909] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1544.452084] env[63593]: INFO nova.compute.manager [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1544.452332] env[63593]: DEBUG oslo.service.loopingcall [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1544.452537] env[63593]: DEBUG nova.compute.manager [-] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1544.452628] env[63593]: DEBUG nova.network.neutron [-] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1544.468017] env[63593]: DEBUG nova.network.neutron [-] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1544.970797] env[63593]: DEBUG nova.network.neutron [-] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1545.473348] env[63593]: INFO nova.compute.manager [-] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] Took 1.02 seconds to deallocate network for instance. [ 1545.476469] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a7217cc-86b9-4546-a010-5fae4c0eafec {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1545.486233] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f57b8c5-b17f-45eb-8e95-afbe72f3d707 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1545.515458] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ac03bed-9fba-4d86-b5f9-5db07d17e968 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1545.522547] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4997d616-4da8-4ae8-8b24-8b5792db52ff {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1545.536226] env[63593]: DEBUG nova.compute.provider_tree [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1546.038961] env[63593]: DEBUG nova.scheduler.client.report [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1546.500515] env[63593]: DEBUG oslo_concurrency.lockutils [None req-8f2c8a9b-2feb-46fa-987c-e2c80d46403c tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 3.188s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1546.501388] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 345.459s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1546.501574] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 4e8f8346-70b0-4ae4-925c-50094ac5a07a] During sync_power_state the instance has a pending task (deleting). Skip. [ 1546.501744] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "4e8f8346-70b0-4ae4-925c-50094ac5a07a" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1546.543964] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.203s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1546.544717] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1547.049572] env[63593]: DEBUG nova.compute.utils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1547.050958] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1547.051142] env[63593]: DEBUG nova.network.neutron [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1547.091123] env[63593]: DEBUG nova.policy [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'af838423630c4726867e594f08d0d76b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2645ec15daad43ff85f00bd59d5293a8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1547.357292] env[63593]: DEBUG nova.network.neutron [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Successfully created port: 987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1547.556778] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1548.564220] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1548.589270] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1548.589519] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1548.589674] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1548.589855] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1548.589999] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1548.590158] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1548.590360] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1548.590514] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1548.590676] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1548.590834] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1548.590999] env[63593]: DEBUG nova.virt.hardware [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1548.591860] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-081b648c-fa2c-46bf-a37d-bd9272017992 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1548.600248] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-183296c1-c216-4154-8f7f-d6238efffa45 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1548.925112] env[63593]: DEBUG nova.network.neutron [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Successfully updated port: 987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1548.934645] env[63593]: DEBUG nova.compute.manager [req-f645274b-055e-4c06-8c4e-6bd0450286c8 req-d01c4be6-777b-4f8e-8b2a-622bf828e19b service nova] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Received event network-vif-plugged-987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1548.934946] env[63593]: DEBUG oslo_concurrency.lockutils [req-f645274b-055e-4c06-8c4e-6bd0450286c8 req-d01c4be6-777b-4f8e-8b2a-622bf828e19b service nova] Acquiring lock "0484e878-6799-4030-9dcc-d62d32317744-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1548.935138] env[63593]: DEBUG oslo_concurrency.lockutils [req-f645274b-055e-4c06-8c4e-6bd0450286c8 req-d01c4be6-777b-4f8e-8b2a-622bf828e19b service nova] Lock "0484e878-6799-4030-9dcc-d62d32317744-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1548.935336] env[63593]: DEBUG oslo_concurrency.lockutils [req-f645274b-055e-4c06-8c4e-6bd0450286c8 req-d01c4be6-777b-4f8e-8b2a-622bf828e19b service nova] Lock "0484e878-6799-4030-9dcc-d62d32317744-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1548.935541] env[63593]: DEBUG nova.compute.manager [req-f645274b-055e-4c06-8c4e-6bd0450286c8 req-d01c4be6-777b-4f8e-8b2a-622bf828e19b service nova] [instance: 0484e878-6799-4030-9dcc-d62d32317744] No waiting events found dispatching network-vif-plugged-987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1548.935725] env[63593]: WARNING nova.compute.manager [req-f645274b-055e-4c06-8c4e-6bd0450286c8 req-d01c4be6-777b-4f8e-8b2a-622bf828e19b service nova] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Received unexpected event network-vif-plugged-987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd for instance with vm_state building and task_state spawning. [ 1549.428071] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquiring lock "refresh_cache-0484e878-6799-4030-9dcc-d62d32317744" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1549.428071] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquired lock "refresh_cache-0484e878-6799-4030-9dcc-d62d32317744" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1549.428071] env[63593]: DEBUG nova.network.neutron [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1549.959131] env[63593]: DEBUG nova.network.neutron [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1550.089970] env[63593]: DEBUG nova.network.neutron [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Updating instance_info_cache with network_info: [{"id": "987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd", "address": "fa:16:3e:b2:40:c5", "network": {"id": "012b5ba7-d45e-4437-9d4b-aa60629c2dbc", "bridge": "br-int", "label": "tempest-ServersNegativeTestJSON-1446325254-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "2645ec15daad43ff85f00bd59d5293a8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "2ee43879-c0b2-47f7-80d0-2c86e3d6d8b5", "external-id": "nsx-vlan-transportzone-151", "segmentation_id": 151, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap987ebb08-6b", "ovs_interfaceid": "987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1550.592645] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Releasing lock "refresh_cache-0484e878-6799-4030-9dcc-d62d32317744" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1550.592992] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Instance network_info: |[{"id": "987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd", "address": "fa:16:3e:b2:40:c5", "network": {"id": "012b5ba7-d45e-4437-9d4b-aa60629c2dbc", "bridge": "br-int", "label": "tempest-ServersNegativeTestJSON-1446325254-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "2645ec15daad43ff85f00bd59d5293a8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "2ee43879-c0b2-47f7-80d0-2c86e3d6d8b5", "external-id": "nsx-vlan-transportzone-151", "segmentation_id": 151, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap987ebb08-6b", "ovs_interfaceid": "987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1550.593459] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:b2:40:c5', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '2ee43879-c0b2-47f7-80d0-2c86e3d6d8b5', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1550.600948] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Creating folder: Project (2645ec15daad43ff85f00bd59d5293a8). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1550.601240] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-025c3d7b-5ad5-4a62-9591-69d02bc6e36f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1550.612668] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Created folder: Project (2645ec15daad43ff85f00bd59d5293a8) in parent group-v291016. [ 1550.612805] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Creating folder: Instances. Parent ref: group-v291088. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1550.613037] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d3dd4555-2832-4f40-9e43-24d093184bbb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1550.621759] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Created folder: Instances in parent group-v291088. [ 1550.621991] env[63593]: DEBUG oslo.service.loopingcall [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1550.622189] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1550.622387] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-6115df9f-01f0-402b-9ef0-7de3d0caadfb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1550.640283] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1550.640283] env[63593]: value = "task-1368062" [ 1550.640283] env[63593]: _type = "Task" [ 1550.640283] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1550.647427] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368062, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1550.959653] env[63593]: DEBUG nova.compute.manager [req-15f49d53-9d92-4774-9609-4f2db0761ebb req-cfb6b6e1-1922-4a76-9070-97b7325bb05d service nova] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Received event network-changed-987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1550.959919] env[63593]: DEBUG nova.compute.manager [req-15f49d53-9d92-4774-9609-4f2db0761ebb req-cfb6b6e1-1922-4a76-9070-97b7325bb05d service nova] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Refreshing instance network info cache due to event network-changed-987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1550.960097] env[63593]: DEBUG oslo_concurrency.lockutils [req-15f49d53-9d92-4774-9609-4f2db0761ebb req-cfb6b6e1-1922-4a76-9070-97b7325bb05d service nova] Acquiring lock "refresh_cache-0484e878-6799-4030-9dcc-d62d32317744" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1550.960318] env[63593]: DEBUG oslo_concurrency.lockutils [req-15f49d53-9d92-4774-9609-4f2db0761ebb req-cfb6b6e1-1922-4a76-9070-97b7325bb05d service nova] Acquired lock "refresh_cache-0484e878-6799-4030-9dcc-d62d32317744" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1550.960491] env[63593]: DEBUG nova.network.neutron [req-15f49d53-9d92-4774-9609-4f2db0761ebb req-cfb6b6e1-1922-4a76-9070-97b7325bb05d service nova] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Refreshing network info cache for port 987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1551.149767] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368062, 'name': CreateVM_Task, 'duration_secs': 0.284867} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1551.149932] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1551.150556] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1551.150719] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1551.151039] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1551.151278] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-bf43a550-65f5-44d2-b8d7-aeb082d48c0c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1551.155425] env[63593]: DEBUG oslo_vmware.api [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Waiting for the task: (returnval){ [ 1551.155425] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52fe9bbe-dfbb-ae53-2e75-02a85fd6998d" [ 1551.155425] env[63593]: _type = "Task" [ 1551.155425] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1551.162577] env[63593]: DEBUG oslo_vmware.api [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52fe9bbe-dfbb-ae53-2e75-02a85fd6998d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1551.665560] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1551.665812] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1551.666050] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1551.703846] env[63593]: DEBUG nova.network.neutron [req-15f49d53-9d92-4774-9609-4f2db0761ebb req-cfb6b6e1-1922-4a76-9070-97b7325bb05d service nova] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Updated VIF entry in instance network info cache for port 987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1551.704199] env[63593]: DEBUG nova.network.neutron [req-15f49d53-9d92-4774-9609-4f2db0761ebb req-cfb6b6e1-1922-4a76-9070-97b7325bb05d service nova] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Updating instance_info_cache with network_info: [{"id": "987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd", "address": "fa:16:3e:b2:40:c5", "network": {"id": "012b5ba7-d45e-4437-9d4b-aa60629c2dbc", "bridge": "br-int", "label": "tempest-ServersNegativeTestJSON-1446325254-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "2645ec15daad43ff85f00bd59d5293a8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "2ee43879-c0b2-47f7-80d0-2c86e3d6d8b5", "external-id": "nsx-vlan-transportzone-151", "segmentation_id": 151, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap987ebb08-6b", "ovs_interfaceid": "987ebb08-6b5f-4ccc-acf6-eaf1bc5e6abd", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1552.206999] env[63593]: DEBUG oslo_concurrency.lockutils [req-15f49d53-9d92-4774-9609-4f2db0761ebb req-cfb6b6e1-1922-4a76-9070-97b7325bb05d service nova] Releasing lock "refresh_cache-0484e878-6799-4030-9dcc-d62d32317744" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1575.012631] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1575.012631] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1577.013715] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1577.014131] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1577.014131] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1577.519175] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.519364] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.519448] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.519573] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.519691] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.519807] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.519923] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.520047] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.520162] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.520282] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1577.520393] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1579.012626] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1579.012943] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1580.014561] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1581.013362] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1582.013171] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1585.564239] env[63593]: WARNING oslo_vmware.rw_handles [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1585.564239] env[63593]: ERROR oslo_vmware.rw_handles [ 1585.564765] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/4577a9ba-f25b-4cd7-9603-71e6d4b34f67/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1585.567506] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1585.567764] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Copying Virtual Disk [datastore2] vmware_temp/4577a9ba-f25b-4cd7-9603-71e6d4b34f67/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/4577a9ba-f25b-4cd7-9603-71e6d4b34f67/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1585.568061] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-35a38c55-8340-4bc2-be12-59424d797630 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1585.576978] env[63593]: DEBUG oslo_vmware.api [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Waiting for the task: (returnval){ [ 1585.576978] env[63593]: value = "task-1368063" [ 1585.576978] env[63593]: _type = "Task" [ 1585.576978] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1585.585237] env[63593]: DEBUG oslo_vmware.api [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Task: {'id': task-1368063, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1586.009309] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1586.013046] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1586.087355] env[63593]: DEBUG oslo_vmware.exceptions [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1586.087642] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1586.088207] env[63593]: ERROR nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1586.088207] env[63593]: Faults: ['InvalidArgument'] [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] Traceback (most recent call last): [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] yield resources [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] self.driver.spawn(context, instance, image_meta, [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] self._fetch_image_if_missing(context, vi) [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] image_cache(vi, tmp_image_ds_loc) [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] vm_util.copy_virtual_disk( [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] session._wait_for_task(vmdk_copy_task) [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] return self.wait_for_task(task_ref) [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] return evt.wait() [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] result = hub.switch() [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] return self.greenlet.switch() [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] self.f(*self.args, **self.kw) [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] raise exceptions.translate_fault(task_info.error) [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] Faults: ['InvalidArgument'] [ 1586.088207] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] [ 1586.089112] env[63593]: INFO nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Terminating instance [ 1586.090055] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1586.090277] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1586.090720] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-378b5b68-c6e8-4aab-8c56-4d19f5d8f5bc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.093017] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1586.093195] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1586.093873] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6ebc7e1-1a8b-48a0-a29e-afd39643e880 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.100229] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1586.100445] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-02d7fa1d-a73d-4508-bf2a-f5e0a6d9459c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.102409] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1586.102579] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1586.103521] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c01d78b4-1d5f-4624-aed2-042cc231af6d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.108194] env[63593]: DEBUG oslo_vmware.api [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Waiting for the task: (returnval){ [ 1586.108194] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5207652f-3232-fe55-07d0-79da3a35348d" [ 1586.108194] env[63593]: _type = "Task" [ 1586.108194] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1586.114631] env[63593]: DEBUG oslo_vmware.api [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5207652f-3232-fe55-07d0-79da3a35348d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1586.169647] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1586.169806] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1586.169972] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Deleting the datastore file [datastore2] 488c4bca-8279-4de7-8259-238497cbd011 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1586.170236] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0b36f479-1c7a-4f7d-af33-7c448492905c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.176197] env[63593]: DEBUG oslo_vmware.api [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Waiting for the task: (returnval){ [ 1586.176197] env[63593]: value = "task-1368065" [ 1586.176197] env[63593]: _type = "Task" [ 1586.176197] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1586.183346] env[63593]: DEBUG oslo_vmware.api [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Task: {'id': task-1368065, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1586.516457] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1586.516671] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1586.516851] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1586.517016] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1586.517899] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5beffd73-785d-4c70-a8d0-cd058cd9d864 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.526184] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4579374-7fe8-4049-a3d8-ae8d9ec90f80 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.539402] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce690c91-35ea-46b7-b82b-6c477c40bd8e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.545778] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4aede3a-9b15-44de-a54c-222a6a88893c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.574949] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181424MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1586.575246] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1586.575294] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1586.617617] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1586.617850] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Creating directory with path [datastore2] vmware_temp/ecc871cd-54fa-4ea9-bbb3-c5323dd2394f/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1586.618067] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-9f2bbec4-8a06-4c4f-b562-d719b28b0909 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.629858] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Created directory with path [datastore2] vmware_temp/ecc871cd-54fa-4ea9-bbb3-c5323dd2394f/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1586.630054] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Fetch image to [datastore2] vmware_temp/ecc871cd-54fa-4ea9-bbb3-c5323dd2394f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1586.630223] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/ecc871cd-54fa-4ea9-bbb3-c5323dd2394f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1586.631184] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4501c0e-3c7d-494c-a391-1f64e6c7676a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.637536] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1ae92cf-bc76-4a2c-9c01-b12854bcc374 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.646546] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c310a64d-1562-4e37-8371-dcd78fdd9076 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.676435] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79ca5ecd-ec0e-4069-87f5-c23911d4918f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.687325] env[63593]: DEBUG oslo_vmware.api [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Task: {'id': task-1368065, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.075135} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1586.687473] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4b58da28-403d-4f25-9da3-ac70f5f0128e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1586.689064] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1586.689258] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1586.689433] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1586.689606] env[63593]: INFO nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1586.692776] env[63593]: DEBUG nova.compute.claims [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1586.692949] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1586.712212] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1586.762740] env[63593]: DEBUG oslo_vmware.rw_handles [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ecc871cd-54fa-4ea9-bbb3-c5323dd2394f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1586.824418] env[63593]: DEBUG oslo_vmware.rw_handles [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1586.824645] env[63593]: DEBUG oslo_vmware.rw_handles [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ecc871cd-54fa-4ea9-bbb3-c5323dd2394f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1587.606093] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 488c4bca-8279-4de7-8259-238497cbd011 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1587.606397] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a55c8243-3413-435e-89cb-6a2bb65b5292 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1587.606436] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1587.606533] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1587.606647] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1587.606759] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1587.606869] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1587.606978] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1587.607098] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1587.607205] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1588.110481] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1588.110700] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1588.110844] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1588.228508] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83704004-07f6-4d24-9f81-3b3e806a6581 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1588.236417] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-024e5e5b-4974-4378-828b-84869a9c9495 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1588.265203] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f871326-d0a2-4b78-b7b0-51f88dde4584 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1588.272394] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-066f6c53-2952-4856-81b2-0349caad1921 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1588.284810] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1588.788275] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1589.295033] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1589.295033] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.719s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1589.295033] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 2.601s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1589.915448] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d61810d-c7ca-4ae0-9b29-717d8a1a4406 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1589.922718] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd1ac383-46e8-46a0-9512-a5934fdf07f7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1589.953345] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-721692a3-2856-4101-8bc5-b72e1fcc613c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1589.960637] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-580493f1-027b-47b3-830b-3f04db016b74 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1589.973436] env[63593]: DEBUG nova.compute.provider_tree [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1590.476906] env[63593]: DEBUG nova.scheduler.client.report [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1590.982021] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.687s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1590.982673] env[63593]: ERROR nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1590.982673] env[63593]: Faults: ['InvalidArgument'] [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] Traceback (most recent call last): [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] self.driver.spawn(context, instance, image_meta, [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] self._fetch_image_if_missing(context, vi) [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] image_cache(vi, tmp_image_ds_loc) [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] vm_util.copy_virtual_disk( [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] session._wait_for_task(vmdk_copy_task) [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] return self.wait_for_task(task_ref) [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] return evt.wait() [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] result = hub.switch() [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] return self.greenlet.switch() [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] self.f(*self.args, **self.kw) [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] raise exceptions.translate_fault(task_info.error) [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] Faults: ['InvalidArgument'] [ 1590.982673] env[63593]: ERROR nova.compute.manager [instance: 488c4bca-8279-4de7-8259-238497cbd011] [ 1590.983693] env[63593]: DEBUG nova.compute.utils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1590.985203] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Build of instance 488c4bca-8279-4de7-8259-238497cbd011 was re-scheduled: A specified parameter was not correct: fileType [ 1590.985203] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1590.985587] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1590.985756] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1590.985934] env[63593]: DEBUG nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1590.986106] env[63593]: DEBUG nova.network.neutron [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1591.740352] env[63593]: DEBUG nova.network.neutron [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1592.243056] env[63593]: INFO nova.compute.manager [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Took 1.26 seconds to deallocate network for instance. [ 1593.279648] env[63593]: INFO nova.scheduler.client.report [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Deleted allocations for instance 488c4bca-8279-4de7-8259-238497cbd011 [ 1593.787693] env[63593]: DEBUG oslo_concurrency.lockutils [None req-66212632-abcc-4dc6-ae14-a6033de999c3 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "488c4bca-8279-4de7-8259-238497cbd011" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 582.101s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1593.789027] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "488c4bca-8279-4de7-8259-238497cbd011" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 392.747s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1593.789157] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 488c4bca-8279-4de7-8259-238497cbd011] During sync_power_state the instance has a pending task (spawning). Skip. [ 1593.789334] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "488c4bca-8279-4de7-8259-238497cbd011" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1593.789765] env[63593]: DEBUG oslo_concurrency.lockutils [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "488c4bca-8279-4de7-8259-238497cbd011" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 386.551s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1593.791028] env[63593]: DEBUG oslo_concurrency.lockutils [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Acquiring lock "488c4bca-8279-4de7-8259-238497cbd011-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1593.791028] env[63593]: DEBUG oslo_concurrency.lockutils [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "488c4bca-8279-4de7-8259-238497cbd011-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1593.791028] env[63593]: DEBUG oslo_concurrency.lockutils [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "488c4bca-8279-4de7-8259-238497cbd011-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1593.793648] env[63593]: INFO nova.compute.manager [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Terminating instance [ 1593.795333] env[63593]: DEBUG nova.compute.manager [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1593.795764] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1593.795764] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5eb9035d-966a-445d-b6cd-2cff0adc5819 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1593.805383] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db54c10a-7732-4f26-a2eb-0939ef4e34df {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1593.831258] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 488c4bca-8279-4de7-8259-238497cbd011 could not be found. [ 1593.831446] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1593.831614] env[63593]: INFO nova.compute.manager [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1593.831835] env[63593]: DEBUG oslo.service.loopingcall [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1593.832052] env[63593]: DEBUG nova.compute.manager [-] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1593.832147] env[63593]: DEBUG nova.network.neutron [-] [instance: 488c4bca-8279-4de7-8259-238497cbd011] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1594.291097] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1594.416699] env[63593]: DEBUG nova.network.neutron [-] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1594.812779] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1594.813080] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1594.814519] env[63593]: INFO nova.compute.claims [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1594.919184] env[63593]: INFO nova.compute.manager [-] [instance: 488c4bca-8279-4de7-8259-238497cbd011] Took 1.09 seconds to deallocate network for instance. [ 1595.943407] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-733f25e1-3dc4-4cd2-a4bc-f0e203ee0713 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1595.945890] env[63593]: DEBUG oslo_concurrency.lockutils [None req-456654d5-f213-4778-a97a-9f99de677425 tempest-ServersNegativeTestMultiTenantJSON-1369553665 tempest-ServersNegativeTestMultiTenantJSON-1369553665-project-member] Lock "488c4bca-8279-4de7-8259-238497cbd011" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.156s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1595.951596] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb7a5ac8-7779-44fd-a296-d9063d842933 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1595.982468] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50c68e6f-1229-4d4e-b56a-21bb2b97ef8a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1595.990702] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f387f59d-9f16-4307-941a-3cb0188e80d2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1596.004581] env[63593]: DEBUG nova.compute.provider_tree [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1596.507703] env[63593]: DEBUG nova.scheduler.client.report [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1597.012912] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.200s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1597.013505] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1597.518896] env[63593]: DEBUG nova.compute.utils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1597.520641] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1597.520787] env[63593]: DEBUG nova.network.neutron [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1597.558837] env[63593]: DEBUG nova.policy [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3bb930b4d1534f3ea3abde038aa60100', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3101ac6769714328b1b7e2c5f371bccc', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1597.801996] env[63593]: DEBUG nova.network.neutron [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Successfully created port: 6e5e3f0a-6976-4ca4-bc63-8a742d9fd342 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1598.023725] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1599.033086] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1599.057833] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1599.057833] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1599.058021] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1599.058107] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1599.058253] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1599.058396] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1599.058595] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1599.058755] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1599.058923] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1599.059092] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1599.059261] env[63593]: DEBUG nova.virt.hardware [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1599.060136] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1d794d5-f3f9-46bd-8c0b-e8af9156d3df {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1599.067895] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab02c3f2-d831-449d-b16f-57a3060a1394 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1599.427285] env[63593]: DEBUG nova.compute.manager [req-77afdbb6-5541-48ff-bd38-1bc3ae532d74 req-12b1e7f2-1561-4308-b0b5-c955d6936da9 service nova] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Received event network-vif-plugged-6e5e3f0a-6976-4ca4-bc63-8a742d9fd342 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1599.427505] env[63593]: DEBUG oslo_concurrency.lockutils [req-77afdbb6-5541-48ff-bd38-1bc3ae532d74 req-12b1e7f2-1561-4308-b0b5-c955d6936da9 service nova] Acquiring lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1599.427729] env[63593]: DEBUG oslo_concurrency.lockutils [req-77afdbb6-5541-48ff-bd38-1bc3ae532d74 req-12b1e7f2-1561-4308-b0b5-c955d6936da9 service nova] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1599.427910] env[63593]: DEBUG oslo_concurrency.lockutils [req-77afdbb6-5541-48ff-bd38-1bc3ae532d74 req-12b1e7f2-1561-4308-b0b5-c955d6936da9 service nova] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1599.429098] env[63593]: DEBUG nova.compute.manager [req-77afdbb6-5541-48ff-bd38-1bc3ae532d74 req-12b1e7f2-1561-4308-b0b5-c955d6936da9 service nova] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] No waiting events found dispatching network-vif-plugged-6e5e3f0a-6976-4ca4-bc63-8a742d9fd342 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1599.429329] env[63593]: WARNING nova.compute.manager [req-77afdbb6-5541-48ff-bd38-1bc3ae532d74 req-12b1e7f2-1561-4308-b0b5-c955d6936da9 service nova] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Received unexpected event network-vif-plugged-6e5e3f0a-6976-4ca4-bc63-8a742d9fd342 for instance with vm_state building and task_state spawning. [ 1599.538925] env[63593]: DEBUG nova.network.neutron [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Successfully updated port: 6e5e3f0a-6976-4ca4-bc63-8a742d9fd342 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1600.042596] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "refresh_cache-5f655ad5-1150-481c-9d34-f47eb8adcb9e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1600.042596] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "refresh_cache-5f655ad5-1150-481c-9d34-f47eb8adcb9e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1600.042596] env[63593]: DEBUG nova.network.neutron [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1600.571461] env[63593]: DEBUG nova.network.neutron [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1600.727946] env[63593]: DEBUG nova.network.neutron [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Updating instance_info_cache with network_info: [{"id": "6e5e3f0a-6976-4ca4-bc63-8a742d9fd342", "address": "fa:16:3e:b8:0a:4d", "network": {"id": "aca8be2a-21d8-4636-8216-f46c0892ddc8", "bridge": "br-int", "label": "tempest-ServersTestJSON-848574930-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.12", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3101ac6769714328b1b7e2c5f371bccc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6e5e3f0a-69", "ovs_interfaceid": "6e5e3f0a-6976-4ca4-bc63-8a742d9fd342", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1601.230889] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "refresh_cache-5f655ad5-1150-481c-9d34-f47eb8adcb9e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1601.231289] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Instance network_info: |[{"id": "6e5e3f0a-6976-4ca4-bc63-8a742d9fd342", "address": "fa:16:3e:b8:0a:4d", "network": {"id": "aca8be2a-21d8-4636-8216-f46c0892ddc8", "bridge": "br-int", "label": "tempest-ServersTestJSON-848574930-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.12", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3101ac6769714328b1b7e2c5f371bccc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6e5e3f0a-69", "ovs_interfaceid": "6e5e3f0a-6976-4ca4-bc63-8a742d9fd342", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1601.231706] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:b8:0a:4d', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '7d377d75-3add-4a15-8691-74b2eb010924', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '6e5e3f0a-6976-4ca4-bc63-8a742d9fd342', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1601.239044] env[63593]: DEBUG oslo.service.loopingcall [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1601.239253] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1601.239466] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-cb5bc90d-7124-48ed-abdd-c8a7d2938b3f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1601.259785] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1601.259785] env[63593]: value = "task-1368066" [ 1601.259785] env[63593]: _type = "Task" [ 1601.259785] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1601.268288] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368066, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1601.452971] env[63593]: DEBUG nova.compute.manager [req-bd822165-2cff-4200-becd-147b7e23045e req-c9915a70-3aed-4cdc-80b7-372df5f13a27 service nova] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Received event network-changed-6e5e3f0a-6976-4ca4-bc63-8a742d9fd342 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1601.453499] env[63593]: DEBUG nova.compute.manager [req-bd822165-2cff-4200-becd-147b7e23045e req-c9915a70-3aed-4cdc-80b7-372df5f13a27 service nova] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Refreshing instance network info cache due to event network-changed-6e5e3f0a-6976-4ca4-bc63-8a742d9fd342. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1601.453757] env[63593]: DEBUG oslo_concurrency.lockutils [req-bd822165-2cff-4200-becd-147b7e23045e req-c9915a70-3aed-4cdc-80b7-372df5f13a27 service nova] Acquiring lock "refresh_cache-5f655ad5-1150-481c-9d34-f47eb8adcb9e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1601.453936] env[63593]: DEBUG oslo_concurrency.lockutils [req-bd822165-2cff-4200-becd-147b7e23045e req-c9915a70-3aed-4cdc-80b7-372df5f13a27 service nova] Acquired lock "refresh_cache-5f655ad5-1150-481c-9d34-f47eb8adcb9e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1601.454297] env[63593]: DEBUG nova.network.neutron [req-bd822165-2cff-4200-becd-147b7e23045e req-c9915a70-3aed-4cdc-80b7-372df5f13a27 service nova] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Refreshing network info cache for port 6e5e3f0a-6976-4ca4-bc63-8a742d9fd342 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1601.769276] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368066, 'name': CreateVM_Task, 'duration_secs': 0.296021} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1601.769475] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1601.769996] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1601.770183] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1601.770505] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1601.770749] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-89414ab5-7630-4ba5-994d-e57ebd418ed6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1601.775414] env[63593]: DEBUG oslo_vmware.api [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 1601.775414] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52c2b4f4-5dbd-b9c1-676a-adf5cf25f911" [ 1601.775414] env[63593]: _type = "Task" [ 1601.775414] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1601.783079] env[63593]: DEBUG oslo_vmware.api [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52c2b4f4-5dbd-b9c1-676a-adf5cf25f911, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1602.144961] env[63593]: DEBUG nova.network.neutron [req-bd822165-2cff-4200-becd-147b7e23045e req-c9915a70-3aed-4cdc-80b7-372df5f13a27 service nova] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Updated VIF entry in instance network info cache for port 6e5e3f0a-6976-4ca4-bc63-8a742d9fd342. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1602.145329] env[63593]: DEBUG nova.network.neutron [req-bd822165-2cff-4200-becd-147b7e23045e req-c9915a70-3aed-4cdc-80b7-372df5f13a27 service nova] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Updating instance_info_cache with network_info: [{"id": "6e5e3f0a-6976-4ca4-bc63-8a742d9fd342", "address": "fa:16:3e:b8:0a:4d", "network": {"id": "aca8be2a-21d8-4636-8216-f46c0892ddc8", "bridge": "br-int", "label": "tempest-ServersTestJSON-848574930-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.12", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3101ac6769714328b1b7e2c5f371bccc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6e5e3f0a-69", "ovs_interfaceid": "6e5e3f0a-6976-4ca4-bc63-8a742d9fd342", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1602.286376] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1602.286760] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1602.287056] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1602.649438] env[63593]: DEBUG oslo_concurrency.lockutils [req-bd822165-2cff-4200-becd-147b7e23045e req-c9915a70-3aed-4cdc-80b7-372df5f13a27 service nova] Releasing lock "refresh_cache-5f655ad5-1150-481c-9d34-f47eb8adcb9e" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1610.990624] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1610.990977] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1616.401702] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1626.130395] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquiring lock "0484e878-6799-4030-9dcc-d62d32317744" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1633.857062] env[63593]: WARNING oslo_vmware.rw_handles [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1633.857062] env[63593]: ERROR oslo_vmware.rw_handles [ 1633.857849] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/ecc871cd-54fa-4ea9-bbb3-c5323dd2394f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1633.859642] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1633.859877] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Copying Virtual Disk [datastore2] vmware_temp/ecc871cd-54fa-4ea9-bbb3-c5323dd2394f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/ecc871cd-54fa-4ea9-bbb3-c5323dd2394f/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1633.860180] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-247311a3-867e-4f08-9e07-c744798a25ea {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1633.869648] env[63593]: DEBUG oslo_vmware.api [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Waiting for the task: (returnval){ [ 1633.869648] env[63593]: value = "task-1368067" [ 1633.869648] env[63593]: _type = "Task" [ 1633.869648] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1633.877942] env[63593]: DEBUG oslo_vmware.api [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Task: {'id': task-1368067, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1634.379872] env[63593]: DEBUG oslo_vmware.exceptions [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1634.380281] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1634.380922] env[63593]: ERROR nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1634.380922] env[63593]: Faults: ['InvalidArgument'] [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Traceback (most recent call last): [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] yield resources [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] self.driver.spawn(context, instance, image_meta, [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] self._fetch_image_if_missing(context, vi) [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] image_cache(vi, tmp_image_ds_loc) [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] vm_util.copy_virtual_disk( [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] session._wait_for_task(vmdk_copy_task) [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] return self.wait_for_task(task_ref) [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] return evt.wait() [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] result = hub.switch() [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] return self.greenlet.switch() [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] self.f(*self.args, **self.kw) [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] raise exceptions.translate_fault(task_info.error) [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Faults: ['InvalidArgument'] [ 1634.380922] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] [ 1634.382579] env[63593]: INFO nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Terminating instance [ 1634.383402] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1634.383669] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1634.383957] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-558b63ce-b5d4-4019-8125-431a67803399 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.386114] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1634.386379] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1634.387158] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-326687c7-3c3e-4a15-91d1-ead154cf6348 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.393600] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1634.393795] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-20fae8b4-60fc-41be-81d5-58a3890862bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.395724] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1634.395885] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1634.396785] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f7c80af3-99b2-4b73-a8ee-21aa3a76e1c4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.400993] env[63593]: DEBUG oslo_vmware.api [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Waiting for the task: (returnval){ [ 1634.400993] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]526b451c-106b-f219-c01d-87a1c44cc2d3" [ 1634.400993] env[63593]: _type = "Task" [ 1634.400993] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1634.410276] env[63593]: DEBUG oslo_vmware.api [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]526b451c-106b-f219-c01d-87a1c44cc2d3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1634.466997] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1634.467241] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1634.467572] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Deleting the datastore file [datastore2] a55c8243-3413-435e-89cb-6a2bb65b5292 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1634.467671] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-a0ed1ddd-a9cf-47ff-aaa3-532525a9d612 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.473169] env[63593]: DEBUG oslo_vmware.api [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Waiting for the task: (returnval){ [ 1634.473169] env[63593]: value = "task-1368069" [ 1634.473169] env[63593]: _type = "Task" [ 1634.473169] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1634.481541] env[63593]: DEBUG oslo_vmware.api [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Task: {'id': task-1368069, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1634.911334] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1634.911610] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Creating directory with path [datastore2] vmware_temp/1a72d333-b66d-4347-a80b-13cac11a4656/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1634.911738] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bd2f4776-0bcb-460b-b5b7-14f937992d05 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.922756] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Created directory with path [datastore2] vmware_temp/1a72d333-b66d-4347-a80b-13cac11a4656/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1634.922942] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Fetch image to [datastore2] vmware_temp/1a72d333-b66d-4347-a80b-13cac11a4656/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1634.923124] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/1a72d333-b66d-4347-a80b-13cac11a4656/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1634.923802] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc051223-986e-4b28-9307-1ec4c8485ebf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.929830] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c4cea1d-8bbe-4278-b2b8-fb162659b2f5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.938333] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3717325d-1fbc-4221-8827-7b2c315b046d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.968415] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e30ab08-2760-45bb-a369-1310ff227d4a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.976227] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-e6fddcb0-8393-4819-90fa-6cf0b4efa065 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.982130] env[63593]: DEBUG oslo_vmware.api [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Task: {'id': task-1368069, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.077169} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1634.982269] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1634.982446] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1634.982611] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1634.982781] env[63593]: INFO nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1634.984875] env[63593]: DEBUG nova.compute.claims [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1634.985053] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1634.985264] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1634.999620] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1635.140906] env[63593]: DEBUG oslo_vmware.rw_handles [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1a72d333-b66d-4347-a80b-13cac11a4656/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1635.201574] env[63593]: DEBUG oslo_vmware.rw_handles [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1635.201720] env[63593]: DEBUG oslo_vmware.rw_handles [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1a72d333-b66d-4347-a80b-13cac11a4656/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1635.617785] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15aa5db3-8ea8-4b58-8ace-c075aa3e57dd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1635.625537] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1f18d11-373a-4d29-8fa5-23e3f08250f5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1635.658999] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c822c96f-cf73-4c08-9d34-610e9674eb36 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1635.666358] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ed3c009-0507-404d-9a11-bbe887e25064 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1635.679459] env[63593]: DEBUG nova.compute.provider_tree [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1636.182511] env[63593]: DEBUG nova.scheduler.client.report [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1636.687499] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.702s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1636.688081] env[63593]: ERROR nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1636.688081] env[63593]: Faults: ['InvalidArgument'] [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Traceback (most recent call last): [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] self.driver.spawn(context, instance, image_meta, [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] self._fetch_image_if_missing(context, vi) [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] image_cache(vi, tmp_image_ds_loc) [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] vm_util.copy_virtual_disk( [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] session._wait_for_task(vmdk_copy_task) [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] return self.wait_for_task(task_ref) [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] return evt.wait() [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] result = hub.switch() [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] return self.greenlet.switch() [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] self.f(*self.args, **self.kw) [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] raise exceptions.translate_fault(task_info.error) [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Faults: ['InvalidArgument'] [ 1636.688081] env[63593]: ERROR nova.compute.manager [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] [ 1636.688964] env[63593]: DEBUG nova.compute.utils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1636.691209] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Build of instance a55c8243-3413-435e-89cb-6a2bb65b5292 was re-scheduled: A specified parameter was not correct: fileType [ 1636.691209] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1636.691936] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1636.692132] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1636.692305] env[63593]: DEBUG nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1636.692469] env[63593]: DEBUG nova.network.neutron [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1637.545607] env[63593]: DEBUG nova.network.neutron [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1638.048478] env[63593]: INFO nova.compute.manager [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Took 1.36 seconds to deallocate network for instance. [ 1639.082924] env[63593]: INFO nova.scheduler.client.report [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Deleted allocations for instance a55c8243-3413-435e-89cb-6a2bb65b5292 [ 1639.297661] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1639.297661] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1639.297849] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1639.592570] env[63593]: DEBUG oslo_concurrency.lockutils [None req-76813921-e087-44a7-930c-f39d51bef56f tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 605.244s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1639.593854] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 438.551s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1639.594067] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] During sync_power_state the instance has a pending task (spawning). Skip. [ 1639.594248] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1639.595471] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 409.300s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1639.595727] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Acquiring lock "a55c8243-3413-435e-89cb-6a2bb65b5292-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1639.595975] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1639.596189] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1639.599019] env[63593]: INFO nova.compute.manager [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Terminating instance [ 1639.600854] env[63593]: DEBUG nova.compute.manager [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1639.601092] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1639.601377] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cc3ed263-96d2-45b3-8921-b23105f8c9a7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1639.612466] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-456b51f5-c531-4ead-a76c-2854da0db9fe {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1639.639767] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a55c8243-3413-435e-89cb-6a2bb65b5292 could not be found. [ 1639.639977] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1639.640206] env[63593]: INFO nova.compute.manager [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1639.640466] env[63593]: DEBUG oslo.service.loopingcall [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1639.640702] env[63593]: DEBUG nova.compute.manager [-] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1639.640814] env[63593]: DEBUG nova.network.neutron [-] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1639.802872] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1639.803070] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1639.803157] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1639.803280] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1639.803444] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1639.803572] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1639.803688] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1639.803804] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1639.803919] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1639.804058] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1639.804277] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1639.804395] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1640.012894] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1640.013160] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1640.095946] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1640.158494] env[63593]: DEBUG nova.network.neutron [-] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1640.617362] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1640.617607] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1640.619109] env[63593]: INFO nova.compute.claims [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1640.660791] env[63593]: INFO nova.compute.manager [-] [instance: a55c8243-3413-435e-89cb-6a2bb65b5292] Took 1.02 seconds to deallocate network for instance. [ 1641.012839] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1641.686630] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e3a06466-f805-444b-98d1-43ba79a5d052 tempest-FloatingIPsAssociationTestJSON-27175813 tempest-FloatingIPsAssociationTestJSON-27175813-project-member] Lock "a55c8243-3413-435e-89cb-6a2bb65b5292" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.091s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1641.742285] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8ac1c28-7e1b-4b02-9090-5fd7638b71d6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1641.749796] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfa1e3ac-3465-46cf-97c4-9276fbb7676f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1641.779234] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba9f0cb0-6488-4084-b7ba-e05b177ccf6e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1641.786120] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4fd68f6-94c6-40df-b12b-ca0be39d07e9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1641.798554] env[63593]: DEBUG nova.compute.provider_tree [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1642.302433] env[63593]: DEBUG nova.scheduler.client.report [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1642.807621] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.190s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1642.808195] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1643.012788] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1643.012788] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1643.315070] env[63593]: DEBUG nova.compute.utils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1643.315527] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1643.315716] env[63593]: DEBUG nova.network.neutron [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1643.363919] env[63593]: DEBUG nova.policy [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a716a2c8efe046e9923bcd6b7956b22a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '60d47be26aac44d198ca43a997485f7c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1643.801663] env[63593]: DEBUG nova.network.neutron [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Successfully created port: 4b9738a3-f59b-401f-9f12-61fc2474402f {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1643.818841] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1644.829325] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1644.853374] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1644.853598] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1644.854169] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1644.854169] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1644.854169] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1644.854314] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1644.854456] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1644.854617] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1644.854782] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1644.854975] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1644.855173] env[63593]: DEBUG nova.virt.hardware [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1644.856060] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9727fa5-c9f5-4c12-a50f-a67cc2744566 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1644.863738] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79d0f162-847c-428a-a18f-c401131bba67 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1645.591063] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1645.593632] env[63593]: DEBUG nova.network.neutron [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Successfully updated port: 4b9738a3-f59b-401f-9f12-61fc2474402f {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1645.598128] env[63593]: DEBUG nova.compute.manager [req-5accd295-ac95-47fc-b1d9-e6e6adb0aafc req-12d62857-6e86-48e5-af40-bb313096d5d9 service nova] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Received event network-vif-plugged-4b9738a3-f59b-401f-9f12-61fc2474402f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1645.598128] env[63593]: DEBUG oslo_concurrency.lockutils [req-5accd295-ac95-47fc-b1d9-e6e6adb0aafc req-12d62857-6e86-48e5-af40-bb313096d5d9 service nova] Acquiring lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1645.598128] env[63593]: DEBUG oslo_concurrency.lockutils [req-5accd295-ac95-47fc-b1d9-e6e6adb0aafc req-12d62857-6e86-48e5-af40-bb313096d5d9 service nova] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1645.598128] env[63593]: DEBUG oslo_concurrency.lockutils [req-5accd295-ac95-47fc-b1d9-e6e6adb0aafc req-12d62857-6e86-48e5-af40-bb313096d5d9 service nova] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1645.598128] env[63593]: DEBUG nova.compute.manager [req-5accd295-ac95-47fc-b1d9-e6e6adb0aafc req-12d62857-6e86-48e5-af40-bb313096d5d9 service nova] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] No waiting events found dispatching network-vif-plugged-4b9738a3-f59b-401f-9f12-61fc2474402f {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1645.598128] env[63593]: WARNING nova.compute.manager [req-5accd295-ac95-47fc-b1d9-e6e6adb0aafc req-12d62857-6e86-48e5-af40-bb313096d5d9 service nova] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Received unexpected event network-vif-plugged-4b9738a3-f59b-401f-9f12-61fc2474402f for instance with vm_state building and task_state spawning. [ 1646.099205] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "refresh_cache-55ecc3e8-678b-4c92-820e-e3b3e01c7530" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1646.099427] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired lock "refresh_cache-55ecc3e8-678b-4c92-820e-e3b3e01c7530" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1646.099536] env[63593]: DEBUG nova.network.neutron [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1646.629467] env[63593]: DEBUG nova.network.neutron [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1646.750384] env[63593]: DEBUG nova.network.neutron [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Updating instance_info_cache with network_info: [{"id": "4b9738a3-f59b-401f-9f12-61fc2474402f", "address": "fa:16:3e:c0:3d:26", "network": {"id": "c440ce66-80a9-4360-813a-816782de8ecd", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-114095660-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "60d47be26aac44d198ca43a997485f7c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f77ff7a1-209c-4f3f-b2a0-fd817741e739", "external-id": "nsx-vlan-transportzone-935", "segmentation_id": 935, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4b9738a3-f5", "ovs_interfaceid": "4b9738a3-f59b-401f-9f12-61fc2474402f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1647.012331] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1647.012581] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1647.205139] env[63593]: DEBUG nova.compute.manager [req-ec3b0488-6558-4889-ab28-96be136ac7b6 req-34aa6ef2-3c04-4321-baa1-a89418b54e3d service nova] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Received event network-changed-4b9738a3-f59b-401f-9f12-61fc2474402f {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1647.205383] env[63593]: DEBUG nova.compute.manager [req-ec3b0488-6558-4889-ab28-96be136ac7b6 req-34aa6ef2-3c04-4321-baa1-a89418b54e3d service nova] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Refreshing instance network info cache due to event network-changed-4b9738a3-f59b-401f-9f12-61fc2474402f. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1647.205527] env[63593]: DEBUG oslo_concurrency.lockutils [req-ec3b0488-6558-4889-ab28-96be136ac7b6 req-34aa6ef2-3c04-4321-baa1-a89418b54e3d service nova] Acquiring lock "refresh_cache-55ecc3e8-678b-4c92-820e-e3b3e01c7530" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1647.252593] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Releasing lock "refresh_cache-55ecc3e8-678b-4c92-820e-e3b3e01c7530" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1647.252891] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Instance network_info: |[{"id": "4b9738a3-f59b-401f-9f12-61fc2474402f", "address": "fa:16:3e:c0:3d:26", "network": {"id": "c440ce66-80a9-4360-813a-816782de8ecd", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-114095660-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "60d47be26aac44d198ca43a997485f7c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f77ff7a1-209c-4f3f-b2a0-fd817741e739", "external-id": "nsx-vlan-transportzone-935", "segmentation_id": 935, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4b9738a3-f5", "ovs_interfaceid": "4b9738a3-f59b-401f-9f12-61fc2474402f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1647.253180] env[63593]: DEBUG oslo_concurrency.lockutils [req-ec3b0488-6558-4889-ab28-96be136ac7b6 req-34aa6ef2-3c04-4321-baa1-a89418b54e3d service nova] Acquired lock "refresh_cache-55ecc3e8-678b-4c92-820e-e3b3e01c7530" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1647.253351] env[63593]: DEBUG nova.network.neutron [req-ec3b0488-6558-4889-ab28-96be136ac7b6 req-34aa6ef2-3c04-4321-baa1-a89418b54e3d service nova] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Refreshing network info cache for port 4b9738a3-f59b-401f-9f12-61fc2474402f {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1647.254804] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:c0:3d:26', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f77ff7a1-209c-4f3f-b2a0-fd817741e739', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '4b9738a3-f59b-401f-9f12-61fc2474402f', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1647.262043] env[63593]: DEBUG oslo.service.loopingcall [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1647.264590] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1647.265030] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-0a8de1e1-b732-425c-bffd-3af37bb615fd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1647.285704] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1647.285704] env[63593]: value = "task-1368070" [ 1647.285704] env[63593]: _type = "Task" [ 1647.285704] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1647.293331] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368070, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1647.466187] env[63593]: DEBUG nova.network.neutron [req-ec3b0488-6558-4889-ab28-96be136ac7b6 req-34aa6ef2-3c04-4321-baa1-a89418b54e3d service nova] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Updated VIF entry in instance network info cache for port 4b9738a3-f59b-401f-9f12-61fc2474402f. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1647.466602] env[63593]: DEBUG nova.network.neutron [req-ec3b0488-6558-4889-ab28-96be136ac7b6 req-34aa6ef2-3c04-4321-baa1-a89418b54e3d service nova] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Updating instance_info_cache with network_info: [{"id": "4b9738a3-f59b-401f-9f12-61fc2474402f", "address": "fa:16:3e:c0:3d:26", "network": {"id": "c440ce66-80a9-4360-813a-816782de8ecd", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-114095660-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "60d47be26aac44d198ca43a997485f7c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f77ff7a1-209c-4f3f-b2a0-fd817741e739", "external-id": "nsx-vlan-transportzone-935", "segmentation_id": 935, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4b9738a3-f5", "ovs_interfaceid": "4b9738a3-f59b-401f-9f12-61fc2474402f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1647.515271] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1647.515523] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1647.515710] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1647.515881] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1647.516783] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62858361-93c8-4897-bef0-1e8d938c06e2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1647.524756] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18244417-5fe9-452c-829c-3d2bdc957ff7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1647.538310] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98165831-1986-4323-80f9-f54686fb8d76 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1647.544556] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ebe08a0-7a8c-4fe6-bdb2-e2d874c296d5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1647.573337] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181422MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1647.573473] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1647.573661] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1647.795912] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368070, 'name': CreateVM_Task, 'duration_secs': 0.278138} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1647.796071] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1647.797044] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1647.797044] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1647.797228] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1647.797476] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f87fa35d-9137-4b66-9e24-043c0dfbfdad {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1647.801934] env[63593]: DEBUG oslo_vmware.api [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 1647.801934] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52acfb6d-7641-001c-d18e-27687e25f45a" [ 1647.801934] env[63593]: _type = "Task" [ 1647.801934] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1647.808613] env[63593]: DEBUG oslo_vmware.api [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52acfb6d-7641-001c-d18e-27687e25f45a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1647.969919] env[63593]: DEBUG oslo_concurrency.lockutils [req-ec3b0488-6558-4889-ab28-96be136ac7b6 req-34aa6ef2-3c04-4321-baa1-a89418b54e3d service nova] Releasing lock "refresh_cache-55ecc3e8-678b-4c92-820e-e3b3e01c7530" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1648.311924] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1648.312229] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1648.312559] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1648.604543] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.604701] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.604866] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.604936] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.605054] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.605172] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.605283] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.605394] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.605504] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.605613] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1648.605802] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1648.605934] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1648.713450] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fd1af86-06c1-4d93-941e-d70e6386d2fa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1648.720606] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e68d0e50-8661-4b49-9cc3-bf75cfe70bbc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1648.749789] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e653277-8380-429c-af64-23d233c6b9a9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1648.756604] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ad78689-1220-4102-b291-843b85cda928 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1648.769097] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1649.272530] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1649.778221] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1649.778622] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.205s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1653.821422] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "3e8b50e4-64fb-4566-b730-9acf5591cbda" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1653.821699] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1665.352563] env[63593]: DEBUG oslo_concurrency.lockutils [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1684.124874] env[63593]: WARNING oslo_vmware.rw_handles [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1684.124874] env[63593]: ERROR oslo_vmware.rw_handles [ 1684.125620] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/1a72d333-b66d-4347-a80b-13cac11a4656/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1684.127675] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1684.127931] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Copying Virtual Disk [datastore2] vmware_temp/1a72d333-b66d-4347-a80b-13cac11a4656/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/1a72d333-b66d-4347-a80b-13cac11a4656/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1684.128269] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-63a5cfa1-bb8e-4bd5-bf5a-4cc5a9750a45 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1684.136237] env[63593]: DEBUG oslo_vmware.api [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Waiting for the task: (returnval){ [ 1684.136237] env[63593]: value = "task-1368071" [ 1684.136237] env[63593]: _type = "Task" [ 1684.136237] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1684.144687] env[63593]: DEBUG oslo_vmware.api [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Task: {'id': task-1368071, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1684.646047] env[63593]: DEBUG oslo_vmware.exceptions [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1684.646142] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1684.646619] env[63593]: ERROR nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1684.646619] env[63593]: Faults: ['InvalidArgument'] [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Traceback (most recent call last): [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] yield resources [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] self.driver.spawn(context, instance, image_meta, [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] self._fetch_image_if_missing(context, vi) [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] image_cache(vi, tmp_image_ds_loc) [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] vm_util.copy_virtual_disk( [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] session._wait_for_task(vmdk_copy_task) [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] return self.wait_for_task(task_ref) [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] return evt.wait() [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] result = hub.switch() [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] return self.greenlet.switch() [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] self.f(*self.args, **self.kw) [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] raise exceptions.translate_fault(task_info.error) [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Faults: ['InvalidArgument'] [ 1684.646619] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] [ 1684.647490] env[63593]: INFO nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Terminating instance [ 1684.648516] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1684.648736] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1684.648970] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-4b4d8053-2427-409e-8207-bdb676d7c729 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1684.651220] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1684.651412] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1684.652129] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfad7e84-2b3d-4460-9fe8-9934265b3c67 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1684.658982] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1684.659203] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-27e55179-10b4-4911-9f17-499caabc5422 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1684.661216] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1684.661388] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1684.662297] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8c4c5073-d666-4b33-9edd-5d8f43d55c08 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1684.666973] env[63593]: DEBUG oslo_vmware.api [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 1684.666973] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]527c39a0-567d-bedc-fc13-bf6fbba8b5e6" [ 1684.666973] env[63593]: _type = "Task" [ 1684.666973] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1684.673718] env[63593]: DEBUG oslo_vmware.api [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]527c39a0-567d-bedc-fc13-bf6fbba8b5e6, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1684.731934] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1684.732263] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1684.732451] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Deleting the datastore file [datastore2] 41e38c8a-3aa1-4524-8820-27ba85187fe8 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1684.732818] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-156bdf04-859b-4ef4-ac9d-c14d1311df81 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1684.739271] env[63593]: DEBUG oslo_vmware.api [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Waiting for the task: (returnval){ [ 1684.739271] env[63593]: value = "task-1368073" [ 1684.739271] env[63593]: _type = "Task" [ 1684.739271] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1684.747130] env[63593]: DEBUG oslo_vmware.api [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Task: {'id': task-1368073, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1685.177594] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1685.177889] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating directory with path [datastore2] vmware_temp/bd9c3758-5b8f-4f13-9e17-2957d73d14d6/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1685.178105] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d85ea685-b8bf-4aa1-a4a0-dfd4908cc9c1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.189033] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Created directory with path [datastore2] vmware_temp/bd9c3758-5b8f-4f13-9e17-2957d73d14d6/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1685.189033] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Fetch image to [datastore2] vmware_temp/bd9c3758-5b8f-4f13-9e17-2957d73d14d6/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1685.189256] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/bd9c3758-5b8f-4f13-9e17-2957d73d14d6/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1685.189933] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40335cbc-1593-4f2e-98e8-ef5476bd7f8e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.196116] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d721fae-8960-4e74-9e6b-67add01cc4a1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.204875] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-199483d6-6589-4e71-be2d-99cb89aeb17a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.234123] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e9a9d1e-e6e4-4de5-8f41-5fae13e6f34b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.239407] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-9d71495b-d607-4096-a1e0-8ecc8878172e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.248863] env[63593]: DEBUG oslo_vmware.api [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Task: {'id': task-1368073, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.074185} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1685.248863] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1685.248863] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1685.248863] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1685.248863] env[63593]: INFO nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1685.250823] env[63593]: DEBUG nova.compute.claims [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1685.250985] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1685.251211] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1685.263348] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1685.313870] env[63593]: DEBUG oslo_vmware.rw_handles [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/bd9c3758-5b8f-4f13-9e17-2957d73d14d6/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1685.374603] env[63593]: DEBUG oslo_vmware.rw_handles [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1685.374815] env[63593]: DEBUG oslo_vmware.rw_handles [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/bd9c3758-5b8f-4f13-9e17-2957d73d14d6/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1685.887335] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df006b73-4edc-489a-b86e-1aef2a8836c7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.894659] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d856218-ddae-48dc-8327-090404f2935c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.923691] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1cb6edd0-14e5-4e8c-a7a6-0cab2e89045b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.930727] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb359b38-7a85-4600-ab9a-bc679263f8ea {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1685.943231] env[63593]: DEBUG nova.compute.provider_tree [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1686.446836] env[63593]: DEBUG nova.scheduler.client.report [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1686.952656] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.701s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1686.953245] env[63593]: ERROR nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1686.953245] env[63593]: Faults: ['InvalidArgument'] [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Traceback (most recent call last): [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] self.driver.spawn(context, instance, image_meta, [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] self._fetch_image_if_missing(context, vi) [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] image_cache(vi, tmp_image_ds_loc) [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] vm_util.copy_virtual_disk( [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] session._wait_for_task(vmdk_copy_task) [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] return self.wait_for_task(task_ref) [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] return evt.wait() [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] result = hub.switch() [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] return self.greenlet.switch() [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] self.f(*self.args, **self.kw) [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] raise exceptions.translate_fault(task_info.error) [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Faults: ['InvalidArgument'] [ 1686.953245] env[63593]: ERROR nova.compute.manager [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] [ 1686.954069] env[63593]: DEBUG nova.compute.utils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1686.955735] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Build of instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 was re-scheduled: A specified parameter was not correct: fileType [ 1686.955735] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1686.956114] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1686.956288] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1686.956459] env[63593]: DEBUG nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1686.956619] env[63593]: DEBUG nova.network.neutron [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1687.693607] env[63593]: DEBUG nova.network.neutron [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1688.197189] env[63593]: INFO nova.compute.manager [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Took 1.24 seconds to deallocate network for instance. [ 1689.231994] env[63593]: INFO nova.scheduler.client.report [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Deleted allocations for instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 [ 1689.740372] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a34df26c-1908-4052-9905-5e21a3aae031 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 613.882s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1689.742244] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 488.699s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1689.742640] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b894e7f9-7b86-49d6-8452-a3533f350e1a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1689.752492] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c3e0b7e-f0eb-405a-9c96-cee484e2172a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1690.247706] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1690.283711] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] During the sync_power process the instance has moved from host None to host cpu-1 [ 1690.283978] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.542s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1690.285033] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 417.495s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1690.285287] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Acquiring lock "41e38c8a-3aa1-4524-8820-27ba85187fe8-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1690.285574] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1690.285746] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1690.287782] env[63593]: INFO nova.compute.manager [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Terminating instance [ 1690.289460] env[63593]: DEBUG nova.compute.manager [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1690.289696] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1690.290174] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e3a50551-b03d-469d-8f20-fbc8f50180ee {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1690.298699] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3d36697-14ad-433d-88d0-260cf7a587a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1690.324038] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 41e38c8a-3aa1-4524-8820-27ba85187fe8 could not be found. [ 1690.324231] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1690.324400] env[63593]: INFO nova.compute.manager [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1690.324622] env[63593]: DEBUG oslo.service.loopingcall [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1690.324861] env[63593]: DEBUG nova.compute.manager [-] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1690.324959] env[63593]: DEBUG nova.network.neutron [-] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1690.770022] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1690.770290] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1690.771767] env[63593]: INFO nova.compute.claims [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1690.841128] env[63593]: DEBUG nova.network.neutron [-] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1691.345246] env[63593]: INFO nova.compute.manager [-] [instance: 41e38c8a-3aa1-4524-8820-27ba85187fe8] Took 1.02 seconds to deallocate network for instance. [ 1691.897910] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1298f9c-04a5-4ade-8b97-bbe394f3f5a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1691.905701] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b05b9095-792e-4cd9-a490-1d3638bcd9d3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1691.936584] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2dc3f4b6-6ec4-4903-b30e-c1cf9dc6dbfb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1691.944376] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ee736b6-ec1b-4e56-a01c-f52d8474b8a5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1691.958676] env[63593]: DEBUG nova.compute.provider_tree [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1692.370530] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ce8936c-fb2a-44d5-824f-671f81dcc302 tempest-AttachVolumeShelveTestJSON-1098821959 tempest-AttachVolumeShelveTestJSON-1098821959-project-member] Lock "41e38c8a-3aa1-4524-8820-27ba85187fe8" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.085s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1692.461259] env[63593]: DEBUG nova.scheduler.client.report [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1692.966723] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.196s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1692.967201] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1693.471917] env[63593]: DEBUG nova.compute.utils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1693.473538] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1693.473538] env[63593]: DEBUG nova.network.neutron [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1693.682900] env[63593]: DEBUG nova.policy [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c0a03b9520a349cb99065ce54ec21927', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '87f6487499be40a9b8bf821e1bd4de80', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1693.931880] env[63593]: DEBUG nova.network.neutron [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Successfully created port: 95d20cf4-9954-407c-a383-abe75a8b8fff {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1693.976618] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1694.985827] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1695.011517] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1695.011807] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1695.012008] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1695.012238] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1695.012383] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1695.012525] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1695.012734] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1695.012890] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1695.013063] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1695.013226] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1695.013393] env[63593]: DEBUG nova.virt.hardware [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1695.014279] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65bb0e2d-2adc-4c6b-8285-8677c0293655 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1695.022768] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-648a5e00-9e2b-4258-a24e-d9f56886352b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1695.448882] env[63593]: DEBUG nova.compute.manager [req-86b7dbd4-f64c-42a9-85eb-9da1f93e7025 req-07fe4cb6-88d0-4753-abb8-05cc7cad4bda service nova] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Received event network-vif-plugged-95d20cf4-9954-407c-a383-abe75a8b8fff {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1695.448882] env[63593]: DEBUG oslo_concurrency.lockutils [req-86b7dbd4-f64c-42a9-85eb-9da1f93e7025 req-07fe4cb6-88d0-4753-abb8-05cc7cad4bda service nova] Acquiring lock "3e8b50e4-64fb-4566-b730-9acf5591cbda-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1695.448882] env[63593]: DEBUG oslo_concurrency.lockutils [req-86b7dbd4-f64c-42a9-85eb-9da1f93e7025 req-07fe4cb6-88d0-4753-abb8-05cc7cad4bda service nova] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1695.448882] env[63593]: DEBUG oslo_concurrency.lockutils [req-86b7dbd4-f64c-42a9-85eb-9da1f93e7025 req-07fe4cb6-88d0-4753-abb8-05cc7cad4bda service nova] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1695.450074] env[63593]: DEBUG nova.compute.manager [req-86b7dbd4-f64c-42a9-85eb-9da1f93e7025 req-07fe4cb6-88d0-4753-abb8-05cc7cad4bda service nova] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] No waiting events found dispatching network-vif-plugged-95d20cf4-9954-407c-a383-abe75a8b8fff {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1695.450445] env[63593]: WARNING nova.compute.manager [req-86b7dbd4-f64c-42a9-85eb-9da1f93e7025 req-07fe4cb6-88d0-4753-abb8-05cc7cad4bda service nova] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Received unexpected event network-vif-plugged-95d20cf4-9954-407c-a383-abe75a8b8fff for instance with vm_state building and task_state spawning. [ 1695.517945] env[63593]: DEBUG nova.network.neutron [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Successfully updated port: 95d20cf4-9954-407c-a383-abe75a8b8fff {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1696.022216] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "refresh_cache-3e8b50e4-64fb-4566-b730-9acf5591cbda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1696.022216] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "refresh_cache-3e8b50e4-64fb-4566-b730-9acf5591cbda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1696.022216] env[63593]: DEBUG nova.network.neutron [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1696.552174] env[63593]: DEBUG nova.network.neutron [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1696.670097] env[63593]: DEBUG nova.network.neutron [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Updating instance_info_cache with network_info: [{"id": "95d20cf4-9954-407c-a383-abe75a8b8fff", "address": "fa:16:3e:86:b6:ea", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap95d20cf4-99", "ovs_interfaceid": "95d20cf4-9954-407c-a383-abe75a8b8fff", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1697.172967] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "refresh_cache-3e8b50e4-64fb-4566-b730-9acf5591cbda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1697.173381] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Instance network_info: |[{"id": "95d20cf4-9954-407c-a383-abe75a8b8fff", "address": "fa:16:3e:86:b6:ea", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap95d20cf4-99", "ovs_interfaceid": "95d20cf4-9954-407c-a383-abe75a8b8fff", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1697.173854] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:86:b6:ea', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a15de394-0367-4921-a5c1-6ac8615e3283', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '95d20cf4-9954-407c-a383-abe75a8b8fff', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1697.181618] env[63593]: DEBUG oslo.service.loopingcall [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1697.181827] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1697.182465] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-bab2f8c8-464c-4ed6-8170-ce91754a1343 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1697.202850] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1697.202850] env[63593]: value = "task-1368074" [ 1697.202850] env[63593]: _type = "Task" [ 1697.202850] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1697.211179] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368074, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1697.476153] env[63593]: DEBUG nova.compute.manager [req-e65a4af3-56be-491b-befa-2499514bb471 req-d28e290f-bf05-4d8a-988d-a8f311322d71 service nova] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Received event network-changed-95d20cf4-9954-407c-a383-abe75a8b8fff {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1697.476423] env[63593]: DEBUG nova.compute.manager [req-e65a4af3-56be-491b-befa-2499514bb471 req-d28e290f-bf05-4d8a-988d-a8f311322d71 service nova] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Refreshing instance network info cache due to event network-changed-95d20cf4-9954-407c-a383-abe75a8b8fff. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1697.476615] env[63593]: DEBUG oslo_concurrency.lockutils [req-e65a4af3-56be-491b-befa-2499514bb471 req-d28e290f-bf05-4d8a-988d-a8f311322d71 service nova] Acquiring lock "refresh_cache-3e8b50e4-64fb-4566-b730-9acf5591cbda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1697.476781] env[63593]: DEBUG oslo_concurrency.lockutils [req-e65a4af3-56be-491b-befa-2499514bb471 req-d28e290f-bf05-4d8a-988d-a8f311322d71 service nova] Acquired lock "refresh_cache-3e8b50e4-64fb-4566-b730-9acf5591cbda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1697.476966] env[63593]: DEBUG nova.network.neutron [req-e65a4af3-56be-491b-befa-2499514bb471 req-d28e290f-bf05-4d8a-988d-a8f311322d71 service nova] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Refreshing network info cache for port 95d20cf4-9954-407c-a383-abe75a8b8fff {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1697.713076] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368074, 'name': CreateVM_Task, 'duration_secs': 0.281785} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1697.713242] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1697.713893] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1697.714068] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1697.714377] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1697.714608] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-be9ff4c9-bf48-4e00-ae1a-f295968ed142 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1697.718813] env[63593]: DEBUG oslo_vmware.api [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 1697.718813] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52d5c86e-1b7a-39d3-cff7-780289c07c72" [ 1697.718813] env[63593]: _type = "Task" [ 1697.718813] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1697.725899] env[63593]: DEBUG oslo_vmware.api [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52d5c86e-1b7a-39d3-cff7-780289c07c72, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1698.197130] env[63593]: DEBUG nova.network.neutron [req-e65a4af3-56be-491b-befa-2499514bb471 req-d28e290f-bf05-4d8a-988d-a8f311322d71 service nova] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Updated VIF entry in instance network info cache for port 95d20cf4-9954-407c-a383-abe75a8b8fff. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1698.198042] env[63593]: DEBUG nova.network.neutron [req-e65a4af3-56be-491b-befa-2499514bb471 req-d28e290f-bf05-4d8a-988d-a8f311322d71 service nova] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Updating instance_info_cache with network_info: [{"id": "95d20cf4-9954-407c-a383-abe75a8b8fff", "address": "fa:16:3e:86:b6:ea", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap95d20cf4-99", "ovs_interfaceid": "95d20cf4-9954-407c-a383-abe75a8b8fff", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1698.229918] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1698.230194] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1698.230401] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1698.700735] env[63593]: DEBUG oslo_concurrency.lockutils [req-e65a4af3-56be-491b-befa-2499514bb471 req-d28e290f-bf05-4d8a-988d-a8f311322d71 service nova] Releasing lock "refresh_cache-3e8b50e4-64fb-4566-b730-9acf5591cbda" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1698.778891] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1698.779063] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1700.014944] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1700.014944] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1700.014944] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1700.518692] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.518692] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.518802] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.518897] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.519052] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.519189] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.519309] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.519426] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.519542] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.519657] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1700.519778] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1701.013122] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1701.013359] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1702.012953] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1705.013066] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1705.013352] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1708.009296] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1709.012658] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1709.516311] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1709.516580] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1709.516753] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1709.516909] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1709.517836] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a55d0b8-d98a-4100-bf71-2bb555d18165 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1709.526095] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-083cc488-0015-491e-b52c-133d6d0fdf39 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1709.539872] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-266a651f-9fef-4785-a4b3-8a88b3949d9a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1709.545858] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a50ab6b0-db76-4b67-a6bc-4b9e470828c8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1709.575458] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181485MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1709.575603] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1709.575814] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1710.607544] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 442bcb9d-533f-4072-b967-791e3bc99643 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.607836] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.607836] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.607974] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.608104] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.608223] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.608336] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.608446] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.608556] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.608667] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 3e8b50e4-64fb-4566-b730-9acf5591cbda actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1710.608908] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1710.609066] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2432MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1710.716277] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04c3c8bc-e9af-4f49-92e8-5c2ae729898e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1710.724379] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1dca878-2d49-4ac6-b90d-3be861721abb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1710.753494] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efaac171-7e31-42f5-bcd5-a61f33c7f496 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1710.760296] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f19f5ea7-44bd-4fce-bb7e-f54bb3826683 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1710.773931] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1711.277171] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1711.781876] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1711.782242] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.206s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1732.751750] env[63593]: WARNING oslo_vmware.rw_handles [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1732.751750] env[63593]: ERROR oslo_vmware.rw_handles [ 1732.752414] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/bd9c3758-5b8f-4f13-9e17-2957d73d14d6/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1732.755306] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1732.755677] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Copying Virtual Disk [datastore2] vmware_temp/bd9c3758-5b8f-4f13-9e17-2957d73d14d6/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/bd9c3758-5b8f-4f13-9e17-2957d73d14d6/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1732.756111] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-70b47a4f-5b04-4227-b617-a1d756147f3a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1732.765532] env[63593]: DEBUG oslo_vmware.api [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 1732.765532] env[63593]: value = "task-1368075" [ 1732.765532] env[63593]: _type = "Task" [ 1732.765532] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1732.777457] env[63593]: DEBUG oslo_vmware.api [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': task-1368075, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1733.276182] env[63593]: DEBUG oslo_vmware.exceptions [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1733.276457] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1733.277029] env[63593]: ERROR nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1733.277029] env[63593]: Faults: ['InvalidArgument'] [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Traceback (most recent call last): [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] yield resources [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] self.driver.spawn(context, instance, image_meta, [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] self._fetch_image_if_missing(context, vi) [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] image_cache(vi, tmp_image_ds_loc) [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] vm_util.copy_virtual_disk( [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] session._wait_for_task(vmdk_copy_task) [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] return self.wait_for_task(task_ref) [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] return evt.wait() [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] result = hub.switch() [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] return self.greenlet.switch() [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] self.f(*self.args, **self.kw) [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] raise exceptions.translate_fault(task_info.error) [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Faults: ['InvalidArgument'] [ 1733.277029] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] [ 1733.277937] env[63593]: INFO nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Terminating instance [ 1733.278912] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1733.279187] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1733.279404] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8d05834f-c01a-4996-bfc4-4ed4822dcac9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.281621] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1733.281816] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1733.282521] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-469aca8d-c821-4015-93cb-123b6cb20996 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.289138] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1733.289363] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-de62ebb9-e831-4aea-bb49-50f868f8bb08 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.291398] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1733.291568] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1733.292496] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c2b433da-fa71-4f68-afb6-2ad24af4e97c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.297186] env[63593]: DEBUG oslo_vmware.api [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Waiting for the task: (returnval){ [ 1733.297186] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52cd3680-8486-e218-9984-0aa066a9ba6b" [ 1733.297186] env[63593]: _type = "Task" [ 1733.297186] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1733.306684] env[63593]: DEBUG oslo_vmware.api [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52cd3680-8486-e218-9984-0aa066a9ba6b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1733.358059] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1733.358059] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1733.358308] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleting the datastore file [datastore2] 442bcb9d-533f-4072-b967-791e3bc99643 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1733.358452] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-43a88f1a-d95c-440c-822f-2eb06cd446ff {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.364637] env[63593]: DEBUG oslo_vmware.api [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 1733.364637] env[63593]: value = "task-1368077" [ 1733.364637] env[63593]: _type = "Task" [ 1733.364637] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1733.372530] env[63593]: DEBUG oslo_vmware.api [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': task-1368077, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1733.807221] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1733.807520] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Creating directory with path [datastore2] vmware_temp/da8e4e21-0752-4eff-ad08-a37f70d0990f/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1733.807692] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-909a87b8-1bda-41cd-808b-5151ac7dc472 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.818410] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Created directory with path [datastore2] vmware_temp/da8e4e21-0752-4eff-ad08-a37f70d0990f/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1733.818592] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Fetch image to [datastore2] vmware_temp/da8e4e21-0752-4eff-ad08-a37f70d0990f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1733.818755] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/da8e4e21-0752-4eff-ad08-a37f70d0990f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1733.819518] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-781317da-7156-45ee-b169-2724dfc72adf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.825985] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bb9b292-548e-44a9-acfc-414c71e78f0b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.835020] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d52cecc1-21d2-47c8-9a83-f9bd80a86f64 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.866078] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ae77946-4244-45c1-8d15-3b840b51808b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.876128] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-b5d55025-369b-4fb5-b3ac-4167894239f3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1733.877721] env[63593]: DEBUG oslo_vmware.api [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': task-1368077, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.065518} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1733.877955] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1733.878164] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1733.878338] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1733.878579] env[63593]: INFO nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1733.880876] env[63593]: DEBUG nova.compute.claims [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1733.881054] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1733.881325] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1733.898912] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1733.950158] env[63593]: DEBUG oslo_vmware.rw_handles [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/da8e4e21-0752-4eff-ad08-a37f70d0990f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1734.009456] env[63593]: DEBUG oslo_vmware.rw_handles [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1734.009667] env[63593]: DEBUG oslo_vmware.rw_handles [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/da8e4e21-0752-4eff-ad08-a37f70d0990f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1734.509279] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c5226b9-5f12-4dc4-8b7a-22e25adcfbef {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1734.517303] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ff948d9-8f51-4b61-95c2-ff5d13984db7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1734.547174] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb7eb996-a06d-40a5-bc2c-ce90cdd1e30b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1734.554293] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-102a6a8a-bded-419f-b287-2a8a16cdbcfa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1734.567053] env[63593]: DEBUG nova.compute.provider_tree [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1735.069930] env[63593]: DEBUG nova.scheduler.client.report [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1735.574706] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.693s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1735.575286] env[63593]: ERROR nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1735.575286] env[63593]: Faults: ['InvalidArgument'] [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Traceback (most recent call last): [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] self.driver.spawn(context, instance, image_meta, [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] self._fetch_image_if_missing(context, vi) [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] image_cache(vi, tmp_image_ds_loc) [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] vm_util.copy_virtual_disk( [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] session._wait_for_task(vmdk_copy_task) [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] return self.wait_for_task(task_ref) [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] return evt.wait() [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] result = hub.switch() [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] return self.greenlet.switch() [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] self.f(*self.args, **self.kw) [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] raise exceptions.translate_fault(task_info.error) [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Faults: ['InvalidArgument'] [ 1735.575286] env[63593]: ERROR nova.compute.manager [instance: 442bcb9d-533f-4072-b967-791e3bc99643] [ 1735.576079] env[63593]: DEBUG nova.compute.utils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1735.577777] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Build of instance 442bcb9d-533f-4072-b967-791e3bc99643 was re-scheduled: A specified parameter was not correct: fileType [ 1735.577777] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1735.578172] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1735.578347] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1735.578515] env[63593]: DEBUG nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1735.578670] env[63593]: DEBUG nova.network.neutron [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1736.317226] env[63593]: DEBUG nova.network.neutron [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1736.820863] env[63593]: INFO nova.compute.manager [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Took 1.24 seconds to deallocate network for instance. [ 1737.854888] env[63593]: INFO nova.scheduler.client.report [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleted allocations for instance 442bcb9d-533f-4072-b967-791e3bc99643 [ 1738.363405] env[63593]: DEBUG oslo_concurrency.lockutils [None req-c4d13c7b-037c-4717-85c8-aaa549051ed3 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "442bcb9d-533f-4072-b967-791e3bc99643" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 662.275s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1738.363732] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "442bcb9d-533f-4072-b967-791e3bc99643" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 465.818s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1738.364023] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "442bcb9d-533f-4072-b967-791e3bc99643-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1738.364378] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "442bcb9d-533f-4072-b967-791e3bc99643-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1738.364480] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "442bcb9d-533f-4072-b967-791e3bc99643-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1738.367192] env[63593]: INFO nova.compute.manager [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Terminating instance [ 1738.368966] env[63593]: DEBUG nova.compute.manager [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1738.369321] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1738.369440] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-96a2814b-c729-4268-9b79-ae53d48149db {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1738.378204] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9abcb6ab-8c17-4a39-9cca-627c278d8776 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1738.405374] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 442bcb9d-533f-4072-b967-791e3bc99643 could not be found. [ 1738.405627] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1738.405751] env[63593]: INFO nova.compute.manager [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1738.405996] env[63593]: DEBUG oslo.service.loopingcall [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1738.406270] env[63593]: DEBUG nova.compute.manager [-] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1738.406365] env[63593]: DEBUG nova.network.neutron [-] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1738.924201] env[63593]: DEBUG nova.network.neutron [-] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1739.427811] env[63593]: INFO nova.compute.manager [-] [instance: 442bcb9d-533f-4072-b967-791e3bc99643] Took 1.02 seconds to deallocate network for instance. [ 1740.452821] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4a3f1826-20d6-4aa4-9796-ad6c960f2223 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "442bcb9d-533f-4072-b967-791e3bc99643" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.089s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1760.782402] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1760.782813] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1760.782813] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1761.287513] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1761.287721] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1761.287779] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1761.287898] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1761.288031] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1761.288157] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1761.288273] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1761.288417] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1761.288537] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1761.288654] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1761.288866] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1761.288990] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1762.012489] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1763.012512] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1763.012864] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1765.013967] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1765.014308] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1765.014394] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1765.014531] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances with incomplete migration {{(pid=63593) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11264}} [ 1767.013330] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1768.510400] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1768.510760] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1769.017220] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1769.017378] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11226}} [ 1769.520239] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] There are 0 instances to clean {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11235}} [ 1771.515585] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1772.018918] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1772.019199] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1772.019378] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1772.019560] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1772.020462] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2edfbc4e-0fd4-4ca1-9b01-4af558d6bcd7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1772.028358] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4186919-a39e-4a56-9ddf-34115910d6a4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1772.041816] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a17d79f5-2fa9-4361-9d53-f005b7f2e461 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1772.048161] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-754a39d0-910e-4767-80ce-940f1bc4edac {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1772.076410] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181486MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1772.076567] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1772.076761] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1773.135731] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1773.135989] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1773.136052] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1773.136137] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1773.136249] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1773.136383] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1773.136471] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1773.136580] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1773.136686] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 3e8b50e4-64fb-4566-b730-9acf5591cbda actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1773.136883] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 9 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1773.137031] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2240MB phys_disk=200GB used_disk=9GB total_vcpus=48 used_vcpus=9 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1773.152748] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing inventories for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 1773.164540] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Updating ProviderTree inventory for provider 7345f17f-3af2-4b0a-9521-0292dc691877 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 1773.164711] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Updating inventory in ProviderTree for provider 7345f17f-3af2-4b0a-9521-0292dc691877 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1773.174342] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing aggregate associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, aggregates: None {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 1773.190803] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing trait associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 1773.282031] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3c187b2-b66d-4b9d-a680-12be16017f63 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1773.289486] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2383e55-0083-4133-960a-ccf36a420fa0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1773.319580] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3dce314-0601-414d-aa95-7497842c6b73 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1773.326295] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c877811a-f00b-4805-950a-e3ba04367d19 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1773.338647] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1773.841648] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1774.346611] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1774.346943] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.270s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1783.838202] env[63593]: WARNING oslo_vmware.rw_handles [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1783.838202] env[63593]: ERROR oslo_vmware.rw_handles [ 1783.838879] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/da8e4e21-0752-4eff-ad08-a37f70d0990f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1783.840669] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1783.840931] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Copying Virtual Disk [datastore2] vmware_temp/da8e4e21-0752-4eff-ad08-a37f70d0990f/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/da8e4e21-0752-4eff-ad08-a37f70d0990f/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1783.841257] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-0098e872-755a-46f5-bff8-7a5f0afab749 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1783.849328] env[63593]: DEBUG oslo_vmware.api [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Waiting for the task: (returnval){ [ 1783.849328] env[63593]: value = "task-1368078" [ 1783.849328] env[63593]: _type = "Task" [ 1783.849328] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1783.856900] env[63593]: DEBUG oslo_vmware.api [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Task: {'id': task-1368078, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1784.360083] env[63593]: DEBUG oslo_vmware.exceptions [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1784.360372] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1784.360953] env[63593]: ERROR nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1784.360953] env[63593]: Faults: ['InvalidArgument'] [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Traceback (most recent call last): [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] yield resources [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self.driver.spawn(context, instance, image_meta, [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self._fetch_image_if_missing(context, vi) [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] image_cache(vi, tmp_image_ds_loc) [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] vm_util.copy_virtual_disk( [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] session._wait_for_task(vmdk_copy_task) [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return self.wait_for_task(task_ref) [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return evt.wait() [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] result = hub.switch() [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return self.greenlet.switch() [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self.f(*self.args, **self.kw) [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] raise exceptions.translate_fault(task_info.error) [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Faults: ['InvalidArgument'] [ 1784.360953] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] [ 1784.362062] env[63593]: INFO nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Terminating instance [ 1784.362835] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1784.363054] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1784.363292] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-6b1ad0e0-a8ae-415d-a794-9f00f696de17 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1784.365248] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "refresh_cache-34867c6c-b8ae-401b-b6f4-ce36bea956ae" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1784.365413] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquired lock "refresh_cache-34867c6c-b8ae-401b-b6f4-ce36bea956ae" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1784.365559] env[63593]: DEBUG nova.network.neutron [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1784.372410] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1784.372577] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1784.373742] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c0ea05fc-dc61-44e6-8954-b0ca42266d5c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1784.379118] env[63593]: DEBUG oslo_vmware.api [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Waiting for the task: (returnval){ [ 1784.379118] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52247205-322a-1317-4d6b-d159f8664c1e" [ 1784.379118] env[63593]: _type = "Task" [ 1784.379118] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1784.386560] env[63593]: DEBUG oslo_vmware.api [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52247205-322a-1317-4d6b-d159f8664c1e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1784.885451] env[63593]: DEBUG nova.network.neutron [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1784.893160] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1784.893409] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Creating directory with path [datastore2] vmware_temp/df17e9e9-db46-492f-9f8a-5f90c911e2e8/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1784.893657] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bfbe5552-8384-4874-b309-b6ca23a4aa2e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1784.912984] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Created directory with path [datastore2] vmware_temp/df17e9e9-db46-492f-9f8a-5f90c911e2e8/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1784.913234] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Fetch image to [datastore2] vmware_temp/df17e9e9-db46-492f-9f8a-5f90c911e2e8/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1784.913420] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/df17e9e9-db46-492f-9f8a-5f90c911e2e8/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1784.914203] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f73095e-e3da-462b-ba49-e2fdfadeeb49 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1784.920852] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f814f8e8-3ffa-48bc-996f-69166898e07b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1784.929663] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9fd5bfb5-f850-4cef-a52f-972f49dcfad8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1784.933582] env[63593]: DEBUG nova.network.neutron [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1784.961789] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c5eb125-360a-41a1-9a9c-8d02139d19d4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1784.967557] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-b1309440-2095-4d93-9f78-e816d1aa7636 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1784.991188] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1785.037540] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/df17e9e9-db46-492f-9f8a-5f90c911e2e8/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1785.098735] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1785.098735] env[63593]: DEBUG oslo_vmware.rw_handles [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/df17e9e9-db46-492f-9f8a-5f90c911e2e8/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1785.436386] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Releasing lock "refresh_cache-34867c6c-b8ae-401b-b6f4-ce36bea956ae" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1785.436752] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1785.436946] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1785.437813] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa666676-fd3f-40b9-a4a2-9c68feca76f4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1785.445557] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1785.445778] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-b9a6059f-de18-4628-a387-7b1fad8a2e41 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1785.479996] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1785.480209] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1785.480381] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Deleting the datastore file [datastore2] 34867c6c-b8ae-401b-b6f4-ce36bea956ae {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1785.480614] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-801d8254-7fea-483f-bfd6-85a4877782a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1785.485986] env[63593]: DEBUG oslo_vmware.api [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Waiting for the task: (returnval){ [ 1785.485986] env[63593]: value = "task-1368080" [ 1785.485986] env[63593]: _type = "Task" [ 1785.485986] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1785.493190] env[63593]: DEBUG oslo_vmware.api [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Task: {'id': task-1368080, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1785.996051] env[63593]: DEBUG oslo_vmware.api [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Task: {'id': task-1368080, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.034457} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1785.996403] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1785.996449] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1785.996576] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1785.996746] env[63593]: INFO nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Took 0.56 seconds to destroy the instance on the hypervisor. [ 1785.996979] env[63593]: DEBUG oslo.service.loopingcall [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1785.997209] env[63593]: DEBUG nova.compute.manager [-] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 1785.999337] env[63593]: DEBUG nova.compute.claims [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1785.999503] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1785.999748] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1786.625858] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07ee1a7f-ee2c-45da-a3ce-6fb3e6632977 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1786.633393] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be26e6f2-14a8-44a8-985c-76a555a3eff5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1786.663117] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-512706c1-7105-402b-973a-b356a5335b53 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1786.670269] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24be086d-e6e9-4eed-ac17-8efd2490620b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1786.683072] env[63593]: DEBUG nova.compute.provider_tree [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1787.186353] env[63593]: DEBUG nova.scheduler.client.report [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1787.691978] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.692s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1787.692570] env[63593]: ERROR nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1787.692570] env[63593]: Faults: ['InvalidArgument'] [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Traceback (most recent call last): [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self.driver.spawn(context, instance, image_meta, [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self._fetch_image_if_missing(context, vi) [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] image_cache(vi, tmp_image_ds_loc) [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] vm_util.copy_virtual_disk( [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] session._wait_for_task(vmdk_copy_task) [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return self.wait_for_task(task_ref) [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return evt.wait() [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] result = hub.switch() [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return self.greenlet.switch() [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self.f(*self.args, **self.kw) [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] raise exceptions.translate_fault(task_info.error) [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Faults: ['InvalidArgument'] [ 1787.692570] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] [ 1787.693495] env[63593]: DEBUG nova.compute.utils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1787.695448] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Build of instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae was re-scheduled: A specified parameter was not correct: fileType [ 1787.695448] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1787.695816] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1787.696058] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "refresh_cache-34867c6c-b8ae-401b-b6f4-ce36bea956ae" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1787.696209] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquired lock "refresh_cache-34867c6c-b8ae-401b-b6f4-ce36bea956ae" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1787.696365] env[63593]: DEBUG nova.network.neutron [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1788.213916] env[63593]: DEBUG nova.network.neutron [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1788.256249] env[63593]: DEBUG nova.network.neutron [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1788.760120] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Releasing lock "refresh_cache-34867c6c-b8ae-401b-b6f4-ce36bea956ae" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1788.760120] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1788.760120] env[63593]: DEBUG nova.compute.manager [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Skipping network deallocation for instance since networking was not requested. {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2285}} [ 1789.792020] env[63593]: INFO nova.scheduler.client.report [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Deleted allocations for instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae [ 1790.299077] env[63593]: DEBUG oslo_concurrency.lockutils [None req-5ab7afbf-7be0-46cf-a501-667fe469ee5c tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "34867c6c-b8ae-401b-b6f4-ce36bea956ae" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 637.813s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1790.299376] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "34867c6c-b8ae-401b-b6f4-ce36bea956ae" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 442.262s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1790.299575] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "34867c6c-b8ae-401b-b6f4-ce36bea956ae-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1790.299777] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "34867c6c-b8ae-401b-b6f4-ce36bea956ae-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1790.299941] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "34867c6c-b8ae-401b-b6f4-ce36bea956ae-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1790.302865] env[63593]: INFO nova.compute.manager [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Terminating instance [ 1790.304497] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquiring lock "refresh_cache-34867c6c-b8ae-401b-b6f4-ce36bea956ae" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1790.304646] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Acquired lock "refresh_cache-34867c6c-b8ae-401b-b6f4-ce36bea956ae" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1790.304803] env[63593]: DEBUG nova.network.neutron [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1790.822965] env[63593]: DEBUG nova.network.neutron [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1790.868454] env[63593]: DEBUG nova.network.neutron [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1791.370754] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Releasing lock "refresh_cache-34867c6c-b8ae-401b-b6f4-ce36bea956ae" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1791.371230] env[63593]: DEBUG nova.compute.manager [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1791.371424] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1791.371723] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-97948b00-105d-44e1-a9dc-08201783771d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1791.381686] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e05b2292-079e-4432-a532-f4fbab418d03 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1791.407908] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 34867c6c-b8ae-401b-b6f4-ce36bea956ae could not be found. [ 1791.408149] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1791.408332] env[63593]: INFO nova.compute.manager [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1791.408591] env[63593]: DEBUG oslo.service.loopingcall [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1791.408842] env[63593]: DEBUG nova.compute.manager [-] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1791.408964] env[63593]: DEBUG nova.network.neutron [-] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1791.501876] env[63593]: DEBUG neutronclient.v2_0.client [-] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=63593) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1791.502147] env[63593]: ERROR nova.network.neutron [-] Neutron client was not able to generate a valid admin token, please verify Neutron admin credential located in nova.conf: neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall [-] Dynamic interval looping call 'oslo_service.loopingcall.RetryDecorator.__call__.._func' failed: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall exception_handler_v20(status_code, error_body) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall raise client_exc(message=error_message, [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall Neutron server returns request_ids: ['req-30aaca92-e918-487f-87af-8ed6f24796c4'] [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall During handling of the above exception, another exception occurred: [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall result = func(*self.args, **self.kw) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall result = f(*args, **kwargs) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 3071, in _deallocate_network_with_retries [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall self._deallocate_network( [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 2291, in _deallocate_network [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall self.network_api.deallocate_for_instance( [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall data = neutron.list_ports(**search_opts) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall return self.list('ports', self.ports_path, retrieve_all, [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall for r in self._pagination(collection, path, **params): [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall res = self.get(path, params=params) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall return self.retry_request("GET", action, body=body, [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall return self.do_request(method, action, body=body, [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall self._handle_fault_response(status_code, replybody, resp) [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1791.502668] env[63593]: ERROR oslo.service.loopingcall [ 1791.504154] env[63593]: ERROR nova.compute.manager [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Failed to deallocate network for instance. Error: Networking client is experiencing an unauthorized exception.: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1792.007951] env[63593]: ERROR nova.compute.manager [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Setting instance vm_state to ERROR: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Traceback (most recent call last): [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] ret = obj(*args, **kwargs) [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] exception_handler_v20(status_code, error_body) [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] raise client_exc(message=error_message, [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Neutron server returns request_ids: ['req-30aaca92-e918-487f-87af-8ed6f24796c4'] [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] During handling of the above exception, another exception occurred: [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Traceback (most recent call last): [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 3341, in do_terminate_instance [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self._delete_instance(context, instance, bdms) [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 3276, in _delete_instance [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self._shutdown_instance(context, instance, bdms) [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 3170, in _shutdown_instance [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self._try_deallocate_network(context, instance, requested_networks) [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 3084, in _try_deallocate_network [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] with excutils.save_and_reraise_exception(): [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self.force_reraise() [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] raise self.value [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 3082, in _try_deallocate_network [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] _deallocate_network_with_retries() [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return evt.wait() [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] result = hub.switch() [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return self.greenlet.switch() [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] result = func(*self.args, **self.kw) [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1792.007951] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] result = f(*args, **kwargs) [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 3071, in _deallocate_network_with_retries [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self._deallocate_network( [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/compute/manager.py", line 2291, in _deallocate_network [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self.network_api.deallocate_for_instance( [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] data = neutron.list_ports(**search_opts) [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] ret = obj(*args, **kwargs) [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return self.list('ports', self.ports_path, retrieve_all, [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] ret = obj(*args, **kwargs) [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] for r in self._pagination(collection, path, **params): [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] res = self.get(path, params=params) [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] ret = obj(*args, **kwargs) [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return self.retry_request("GET", action, body=body, [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] ret = obj(*args, **kwargs) [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] return self.do_request(method, action, body=body, [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] ret = obj(*args, **kwargs) [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] self._handle_fault_response(status_code, replybody, resp) [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1792.009567] env[63593]: ERROR nova.compute.manager [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] [ 1792.512717] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Lock "34867c6c-b8ae-401b-b6f4-ce36bea956ae" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.213s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1794.023229] env[63593]: INFO nova.compute.manager [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] [instance: 34867c6c-b8ae-401b-b6f4-ce36bea956ae] Successfully reverted task state from None on failure for instance. [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server [None req-e7415342-13e7-4814-a3b3-97ef8d3e6cf7 tempest-ServerShowV254Test-155527527 tempest-ServerShowV254Test-155527527-project-member] Exception during message handling: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server exception_handler_v20(status_code, error_body) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server raise client_exc(message=error_message, [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server Neutron server returns request_ids: ['req-30aaca92-e918-487f-87af-8ed6f24796c4'] [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/server.py", line 172, in _process_incoming [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 166, in decorated_function [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 157, in decorated_function [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/utils.py", line 1453, in decorated_function [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 213, in decorated_function [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 203, in decorated_function [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3353, in terminate_instance [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server do_terminate_instance(instance, bdms) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py", line 412, in inner [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server return f(*args, **kwargs) [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3348, in do_terminate_instance [ 1794.027080] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3341, in do_terminate_instance [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server self._delete_instance(context, instance, bdms) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3276, in _delete_instance [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server self._shutdown_instance(context, instance, bdms) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3170, in _shutdown_instance [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server self._try_deallocate_network(context, instance, requested_networks) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3084, in _try_deallocate_network [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server raise self.value [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3082, in _try_deallocate_network [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server _deallocate_network_with_retries() [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server return evt.wait() [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server result = hub.switch() [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server return self.greenlet.switch() [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server result = func(*self.args, **self.kw) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server result = f(*args, **kwargs) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3071, in _deallocate_network_with_retries [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server self._deallocate_network( [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2291, in _deallocate_network [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server self.network_api.deallocate_for_instance( [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server data = neutron.list_ports(**search_opts) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server return self.list('ports', self.ports_path, retrieve_all, [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server for r in self._pagination(collection, path, **params): [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server res = self.get(path, params=params) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server return self.retry_request("GET", action, body=body, [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server return self.do_request(method, action, body=body, [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1794.028705] env[63593]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1794.030166] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1794.030166] env[63593]: ERROR oslo_messaging.rpc.server self._handle_fault_response(status_code, replybody, resp) [ 1794.030166] env[63593]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1794.030166] env[63593]: ERROR oslo_messaging.rpc.server raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1794.030166] env[63593]: ERROR oslo_messaging.rpc.server nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1794.030166] env[63593]: ERROR oslo_messaging.rpc.server [ 1806.789291] env[63593]: DEBUG oslo_concurrency.lockutils [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1816.456432] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquiring lock "606babde-3951-4b7a-b4bd-f7c22ab09195" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1816.456949] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1816.959455] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1817.482139] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1817.482410] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1817.484131] env[63593]: INFO nova.compute.claims [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1818.593673] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afdda504-c684-48de-b9dc-8451c371d1f2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1818.601238] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f23cc602-b031-4942-a070-a87626e44231 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1818.631231] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d39de03-df9d-4588-af81-1847805682d0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1818.637771] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cabfe76c-429c-4f85-8a67-345abdbc807d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1818.650219] env[63593]: DEBUG nova.compute.provider_tree [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1819.153609] env[63593]: DEBUG nova.scheduler.client.report [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1819.658612] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.176s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1819.659194] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1820.164149] env[63593]: DEBUG nova.compute.utils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1820.165545] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1820.165712] env[63593]: DEBUG nova.network.neutron [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1820.203281] env[63593]: DEBUG nova.policy [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '97e91591a7c64b4a99e87d069370aa0f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c686ce2421804c10988de034da7001cf', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1820.438066] env[63593]: DEBUG nova.network.neutron [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Successfully created port: db42a49f-becd-421e-b0b4-2a72665f4aff {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1820.669565] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1821.342285] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1821.342442] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1821.342567] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1821.679364] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1821.705233] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1821.705460] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1821.705683] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1821.705781] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1821.705922] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1821.706080] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1821.706285] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1821.706457] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1821.706591] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1821.706746] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1821.706911] env[63593]: DEBUG nova.virt.hardware [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1821.707847] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0830c7cc-45f3-4114-81f3-820dd6b92dc6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.715834] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ffcf495-24f2-44a3-bfc1-94e6c0221263 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1821.847492] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1821.847687] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1821.847789] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1821.847910] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1821.848046] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1821.848176] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1821.848293] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1821.848414] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1821.848532] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1821.848651] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1821.848885] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_power_states {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1822.159117] env[63593]: DEBUG nova.compute.manager [req-8ccaa3e3-943d-4fbd-8939-6233b9e82174 req-7dfff0cd-be20-417b-995e-55fe8025b747 service nova] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Received event network-vif-plugged-db42a49f-becd-421e-b0b4-2a72665f4aff {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1822.159117] env[63593]: DEBUG oslo_concurrency.lockutils [req-8ccaa3e3-943d-4fbd-8939-6233b9e82174 req-7dfff0cd-be20-417b-995e-55fe8025b747 service nova] Acquiring lock "606babde-3951-4b7a-b4bd-f7c22ab09195-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.159117] env[63593]: DEBUG oslo_concurrency.lockutils [req-8ccaa3e3-943d-4fbd-8939-6233b9e82174 req-7dfff0cd-be20-417b-995e-55fe8025b747 service nova] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1822.159117] env[63593]: DEBUG oslo_concurrency.lockutils [req-8ccaa3e3-943d-4fbd-8939-6233b9e82174 req-7dfff0cd-be20-417b-995e-55fe8025b747 service nova] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1822.159117] env[63593]: DEBUG nova.compute.manager [req-8ccaa3e3-943d-4fbd-8939-6233b9e82174 req-7dfff0cd-be20-417b-995e-55fe8025b747 service nova] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] No waiting events found dispatching network-vif-plugged-db42a49f-becd-421e-b0b4-2a72665f4aff {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1822.159117] env[63593]: WARNING nova.compute.manager [req-8ccaa3e3-943d-4fbd-8939-6233b9e82174 req-7dfff0cd-be20-417b-995e-55fe8025b747 service nova] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Received unexpected event network-vif-plugged-db42a49f-becd-421e-b0b4-2a72665f4aff for instance with vm_state building and task_state spawning. [ 1822.248336] env[63593]: DEBUG nova.network.neutron [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Successfully updated port: db42a49f-becd-421e-b0b4-2a72665f4aff {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1822.353594] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Getting list of instances from cluster (obj){ [ 1822.353594] env[63593]: value = "domain-c8" [ 1822.353594] env[63593]: _type = "ClusterComputeResource" [ 1822.353594] env[63593]: } {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 1822.354650] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e544761-2ee7-46cb-ae2f-be07b243733f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1822.369256] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Got total of 8 instances {{(pid=63593) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 1822.369538] env[63593]: WARNING nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] While synchronizing instance power states, found 9 instances in the database and 8 instances on the hypervisor. [ 1822.369619] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid fcfd0672-c583-4cb0-9e39-37b6e7cdb68e {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1822.369757] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1822.369908] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid a1b46566-4db2-4086-906a-7ba32b9a2813 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1822.370105] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid abaeda1e-4ad8-4553-9fb0-f94e2cd98059 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1822.370285] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 0484e878-6799-4030-9dcc-d62d32317744 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1822.370437] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 5f655ad5-1150-481c-9d34-f47eb8adcb9e {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1822.370582] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 55ecc3e8-678b-4c92-820e-e3b3e01c7530 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1822.370725] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 3e8b50e4-64fb-4566-b730-9acf5591cbda {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1822.370869] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Triggering sync for uuid 606babde-3951-4b7a-b4bd-f7c22ab09195 {{(pid=63593) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10336}} [ 1822.371344] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.371775] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.372016] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "a1b46566-4db2-4086-906a-7ba32b9a2813" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.372330] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.372559] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "0484e878-6799-4030-9dcc-d62d32317744" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.372754] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.372944] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.373190] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "3e8b50e4-64fb-4566-b730-9acf5591cbda" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.373417] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "606babde-3951-4b7a-b4bd-f7c22ab09195" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1822.373594] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1822.373725] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1822.750914] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquiring lock "refresh_cache-606babde-3951-4b7a-b4bd-f7c22ab09195" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1822.751166] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquired lock "refresh_cache-606babde-3951-4b7a-b4bd-f7c22ab09195" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1822.751241] env[63593]: DEBUG nova.network.neutron [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1823.280895] env[63593]: DEBUG nova.network.neutron [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1823.425838] env[63593]: DEBUG nova.network.neutron [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Updating instance_info_cache with network_info: [{"id": "db42a49f-becd-421e-b0b4-2a72665f4aff", "address": "fa:16:3e:b2:dd:c1", "network": {"id": "a3addc40-53f3-4459-936a-3abd563f775a", "bridge": "br-int", "label": "tempest-InstanceActionsV221TestJSON-430625411-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c686ce2421804c10988de034da7001cf", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fb224918-e294-4b76-80f9-2fa0031b7dc2", "external-id": "nsx-vlan-transportzone-876", "segmentation_id": 876, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapdb42a49f-be", "ovs_interfaceid": "db42a49f-becd-421e-b0b4-2a72665f4aff", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1823.537308] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1823.927822] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Releasing lock "refresh_cache-606babde-3951-4b7a-b4bd-f7c22ab09195" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1823.927822] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Instance network_info: |[{"id": "db42a49f-becd-421e-b0b4-2a72665f4aff", "address": "fa:16:3e:b2:dd:c1", "network": {"id": "a3addc40-53f3-4459-936a-3abd563f775a", "bridge": "br-int", "label": "tempest-InstanceActionsV221TestJSON-430625411-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c686ce2421804c10988de034da7001cf", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fb224918-e294-4b76-80f9-2fa0031b7dc2", "external-id": "nsx-vlan-transportzone-876", "segmentation_id": 876, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapdb42a49f-be", "ovs_interfaceid": "db42a49f-becd-421e-b0b4-2a72665f4aff", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1823.927822] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:b2:dd:c1', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'fb224918-e294-4b76-80f9-2fa0031b7dc2', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'db42a49f-becd-421e-b0b4-2a72665f4aff', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1823.935763] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Creating folder: Project (c686ce2421804c10988de034da7001cf). Parent ref: group-v291016. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1823.936073] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-09f389d8-cae3-405a-8d09-b4fd1d1097b1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1823.948230] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Created folder: Project (c686ce2421804c10988de034da7001cf) in parent group-v291016. [ 1823.948469] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Creating folder: Instances. Parent ref: group-v291094. {{(pid=63593) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1823.948728] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0a6595e9-df67-4a63-bfb6-bb3bfa3fdf11 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1823.957560] env[63593]: INFO nova.virt.vmwareapi.vm_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Created folder: Instances in parent group-v291094. [ 1823.957820] env[63593]: DEBUG oslo.service.loopingcall [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1823.958045] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1823.958270] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-c4724c7c-7610-4e28-966d-93ae691bfcfb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1823.976398] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1823.976398] env[63593]: value = "task-1368083" [ 1823.976398] env[63593]: _type = "Task" [ 1823.976398] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1823.987408] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368083, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1824.012981] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1824.186639] env[63593]: DEBUG nova.compute.manager [req-84611495-4154-4b2c-b3a8-90d80bb565cf req-d5e0066b-0f46-4b88-a9d5-645c3ec2ac21 service nova] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Received event network-changed-db42a49f-becd-421e-b0b4-2a72665f4aff {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1824.186875] env[63593]: DEBUG nova.compute.manager [req-84611495-4154-4b2c-b3a8-90d80bb565cf req-d5e0066b-0f46-4b88-a9d5-645c3ec2ac21 service nova] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Refreshing instance network info cache due to event network-changed-db42a49f-becd-421e-b0b4-2a72665f4aff. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1824.187109] env[63593]: DEBUG oslo_concurrency.lockutils [req-84611495-4154-4b2c-b3a8-90d80bb565cf req-d5e0066b-0f46-4b88-a9d5-645c3ec2ac21 service nova] Acquiring lock "refresh_cache-606babde-3951-4b7a-b4bd-f7c22ab09195" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1824.187254] env[63593]: DEBUG oslo_concurrency.lockutils [req-84611495-4154-4b2c-b3a8-90d80bb565cf req-d5e0066b-0f46-4b88-a9d5-645c3ec2ac21 service nova] Acquired lock "refresh_cache-606babde-3951-4b7a-b4bd-f7c22ab09195" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1824.187423] env[63593]: DEBUG nova.network.neutron [req-84611495-4154-4b2c-b3a8-90d80bb565cf req-d5e0066b-0f46-4b88-a9d5-645c3ec2ac21 service nova] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Refreshing network info cache for port db42a49f-becd-421e-b0b4-2a72665f4aff {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1824.485567] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368083, 'name': CreateVM_Task, 'duration_secs': 0.307826} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1824.485743] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1824.486429] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1824.486585] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1824.486933] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1824.487650] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-0304d166-3672-4790-81d5-1fe346891dc4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1824.492281] env[63593]: DEBUG oslo_vmware.api [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Waiting for the task: (returnval){ [ 1824.492281] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5291dc15-88da-daad-e903-652a70205a57" [ 1824.492281] env[63593]: _type = "Task" [ 1824.492281] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1824.500743] env[63593]: DEBUG oslo_vmware.api [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5291dc15-88da-daad-e903-652a70205a57, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1824.861429] env[63593]: DEBUG nova.network.neutron [req-84611495-4154-4b2c-b3a8-90d80bb565cf req-d5e0066b-0f46-4b88-a9d5-645c3ec2ac21 service nova] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Updated VIF entry in instance network info cache for port db42a49f-becd-421e-b0b4-2a72665f4aff. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1824.861775] env[63593]: DEBUG nova.network.neutron [req-84611495-4154-4b2c-b3a8-90d80bb565cf req-d5e0066b-0f46-4b88-a9d5-645c3ec2ac21 service nova] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Updating instance_info_cache with network_info: [{"id": "db42a49f-becd-421e-b0b4-2a72665f4aff", "address": "fa:16:3e:b2:dd:c1", "network": {"id": "a3addc40-53f3-4459-936a-3abd563f775a", "bridge": "br-int", "label": "tempest-InstanceActionsV221TestJSON-430625411-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c686ce2421804c10988de034da7001cf", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fb224918-e294-4b76-80f9-2fa0031b7dc2", "external-id": "nsx-vlan-transportzone-876", "segmentation_id": 876, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapdb42a49f-be", "ovs_interfaceid": "db42a49f-becd-421e-b0b4-2a72665f4aff", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1825.001988] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1825.002326] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1825.002526] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1825.011993] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1825.012202] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1825.364516] env[63593]: DEBUG oslo_concurrency.lockutils [req-84611495-4154-4b2c-b3a8-90d80bb565cf req-d5e0066b-0f46-4b88-a9d5-645c3ec2ac21 service nova] Releasing lock "refresh_cache-606babde-3951-4b7a-b4bd-f7c22ab09195" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1826.012785] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1829.008330] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1830.646264] env[63593]: WARNING oslo_vmware.rw_handles [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1830.646264] env[63593]: ERROR oslo_vmware.rw_handles [ 1830.647079] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/df17e9e9-db46-492f-9f8a-5f90c911e2e8/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1830.648809] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1830.649081] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Copying Virtual Disk [datastore2] vmware_temp/df17e9e9-db46-492f-9f8a-5f90c911e2e8/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/df17e9e9-db46-492f-9f8a-5f90c911e2e8/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1830.649431] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-23cf6b97-a4eb-450e-9182-e24438afa74b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1830.657015] env[63593]: DEBUG oslo_vmware.api [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Waiting for the task: (returnval){ [ 1830.657015] env[63593]: value = "task-1368084" [ 1830.657015] env[63593]: _type = "Task" [ 1830.657015] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1830.664426] env[63593]: DEBUG oslo_vmware.api [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Task: {'id': task-1368084, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1831.167480] env[63593]: DEBUG oslo_vmware.exceptions [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1831.167760] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1831.168353] env[63593]: ERROR nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1831.168353] env[63593]: Faults: ['InvalidArgument'] [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Traceback (most recent call last): [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] yield resources [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] self.driver.spawn(context, instance, image_meta, [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] self._fetch_image_if_missing(context, vi) [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] image_cache(vi, tmp_image_ds_loc) [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] vm_util.copy_virtual_disk( [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] session._wait_for_task(vmdk_copy_task) [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] return self.wait_for_task(task_ref) [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] return evt.wait() [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] result = hub.switch() [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] return self.greenlet.switch() [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] self.f(*self.args, **self.kw) [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] raise exceptions.translate_fault(task_info.error) [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Faults: ['InvalidArgument'] [ 1831.168353] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] [ 1831.169527] env[63593]: INFO nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Terminating instance [ 1831.170902] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1831.171139] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1831.171798] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1831.172014] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1831.172322] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-68e735b3-86fd-4583-b6da-c9d00392b345 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.174523] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f147498-484f-4169-aa23-5f8c40078883 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.181385] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1831.181610] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-c6978994-93c6-458d-91bd-13343340d926 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.183621] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1831.183792] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1831.184701] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f8f5a594-71a4-4548-a346-0cc416556a4e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.189009] env[63593]: DEBUG oslo_vmware.api [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 1831.189009] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5295c087-9f0b-0da4-7fae-2d7a809cbd6c" [ 1831.189009] env[63593]: _type = "Task" [ 1831.189009] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1831.195797] env[63593]: DEBUG oslo_vmware.api [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5295c087-9f0b-0da4-7fae-2d7a809cbd6c, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1831.252918] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1831.253095] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1831.253268] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Deleting the datastore file [datastore2] fcfd0672-c583-4cb0-9e39-37b6e7cdb68e {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1831.253530] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ed1b1804-f1e8-4db2-b19a-d22f5a4976cc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.259155] env[63593]: DEBUG oslo_vmware.api [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Waiting for the task: (returnval){ [ 1831.259155] env[63593]: value = "task-1368086" [ 1831.259155] env[63593]: _type = "Task" [ 1831.259155] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1831.266302] env[63593]: DEBUG oslo_vmware.api [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Task: {'id': task-1368086, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1831.698927] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1831.699222] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating directory with path [datastore2] vmware_temp/741eaeb6-c741-41cc-814a-1680f89b1f04/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1831.699463] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-493047c8-2340-4a5e-b564-f17fdaba660d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.710306] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Created directory with path [datastore2] vmware_temp/741eaeb6-c741-41cc-814a-1680f89b1f04/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1831.710512] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Fetch image to [datastore2] vmware_temp/741eaeb6-c741-41cc-814a-1680f89b1f04/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1831.710678] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/741eaeb6-c741-41cc-814a-1680f89b1f04/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1831.711375] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65d3bb64-7408-4785-944e-089ba3ed462c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.717475] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1797560c-1a41-43a0-932f-5e21674ad021 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.725992] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab315d23-7bb5-49ae-8dde-e8add1636118 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.756434] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f71adfe-f8af-4353-9e0f-34e128affd97 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.764121] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-8619c6b6-d179-490b-979c-2f984cd4e417 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1831.768139] env[63593]: DEBUG oslo_vmware.api [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Task: {'id': task-1368086, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.065797} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1831.768637] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1831.768814] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1831.768981] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1831.769174] env[63593]: INFO nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1831.771272] env[63593]: DEBUG nova.compute.claims [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1831.771456] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1831.771679] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1831.791098] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1831.843241] env[63593]: DEBUG oslo_vmware.rw_handles [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/741eaeb6-c741-41cc-814a-1680f89b1f04/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1831.903678] env[63593]: DEBUG oslo_vmware.rw_handles [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1831.903867] env[63593]: DEBUG oslo_vmware.rw_handles [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/741eaeb6-c741-41cc-814a-1680f89b1f04/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1832.383597] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9ec82e5-f877-4b9c-887b-8ab4e888d2c7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1832.391306] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78669285-662e-46fe-8f68-6f57a3b993bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1832.422940] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b87c4af5-c95c-487b-9b1c-94c972fe2704 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1832.430370] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a919b9cd-f02e-464c-9232-b29eb9846718 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1832.443976] env[63593]: DEBUG nova.compute.provider_tree [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1832.947217] env[63593]: DEBUG nova.scheduler.client.report [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1833.012939] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1833.452019] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.680s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1833.452564] env[63593]: ERROR nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1833.452564] env[63593]: Faults: ['InvalidArgument'] [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Traceback (most recent call last): [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] self.driver.spawn(context, instance, image_meta, [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] self._fetch_image_if_missing(context, vi) [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] image_cache(vi, tmp_image_ds_loc) [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] vm_util.copy_virtual_disk( [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] session._wait_for_task(vmdk_copy_task) [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] return self.wait_for_task(task_ref) [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] return evt.wait() [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] result = hub.switch() [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] return self.greenlet.switch() [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] self.f(*self.args, **self.kw) [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] raise exceptions.translate_fault(task_info.error) [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Faults: ['InvalidArgument'] [ 1833.452564] env[63593]: ERROR nova.compute.manager [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] [ 1833.453858] env[63593]: DEBUG nova.compute.utils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1833.455211] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Build of instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e was re-scheduled: A specified parameter was not correct: fileType [ 1833.455211] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1833.455576] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1833.455747] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1833.455914] env[63593]: DEBUG nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1833.456086] env[63593]: DEBUG nova.network.neutron [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1833.515901] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1833.516213] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1833.516425] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1833.516618] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1833.517636] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7998b34a-404b-4cef-9ba3-a4a5c627ab45 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1833.526190] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98beefd7-16eb-4400-998e-3c36406c48b5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1833.539733] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb94da24-06a0-48cc-8c5e-bb4380559c1a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1833.545860] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3815f22c-f2ba-4ce5-aafd-0fab01997131 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1833.573743] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181469MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1833.573891] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1833.574096] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1834.266743] env[63593]: DEBUG nova.network.neutron [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1834.769697] env[63593]: INFO nova.compute.manager [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Took 1.31 seconds to deallocate network for instance. [ 1835.111786] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1835.111965] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1835.112099] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1835.112219] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1835.112334] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1835.112445] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1835.112556] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1835.112665] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 3e8b50e4-64fb-4566-b730-9acf5591cbda actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1835.112774] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 606babde-3951-4b7a-b4bd-f7c22ab09195 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1835.112968] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 8 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1835.113115] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2048MB phys_disk=200GB used_disk=8GB total_vcpus=48 used_vcpus=8 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1835.215769] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cc063e2-aa73-4439-91c2-2e9fefd9503f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.223363] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d619f539-b0ca-453c-838c-a72ef08f5acc {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.251658] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cccf6e52-98c6-4cd2-9a05-b7c2900a3a1f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.258484] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b12a922-0392-4bf2-a15c-a48e8ff9dab4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1835.270926] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1835.775104] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1835.798677] env[63593]: INFO nova.scheduler.client.report [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Deleted allocations for instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e [ 1836.282073] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1836.282430] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.708s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1836.307121] env[63593]: DEBUG oslo_concurrency.lockutils [None req-2addac20-495c-462f-aac0-0de0e3c4043e tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 628.171s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1836.307335] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 431.644s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1836.307560] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Acquiring lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1836.307774] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1836.307938] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1836.311066] env[63593]: INFO nova.compute.manager [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Terminating instance [ 1836.312874] env[63593]: DEBUG nova.compute.manager [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1836.313084] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1836.313346] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8bd58b66-51b0-432e-9306-908b8412fceb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1836.322102] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c86cb67-ad42-4e15-94a9-1f12dca1d232 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1836.348324] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fcfd0672-c583-4cb0-9e39-37b6e7cdb68e could not be found. [ 1836.348524] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1836.348700] env[63593]: INFO nova.compute.manager [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1836.348937] env[63593]: DEBUG oslo.service.loopingcall [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1836.349173] env[63593]: DEBUG nova.compute.manager [-] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1836.349287] env[63593]: DEBUG nova.network.neutron [-] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1836.866589] env[63593]: DEBUG nova.network.neutron [-] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1837.370352] env[63593]: INFO nova.compute.manager [-] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] Took 1.02 seconds to deallocate network for instance. [ 1838.396019] env[63593]: DEBUG oslo_concurrency.lockutils [None req-db64c0bd-0ad7-45b0-b573-3818d622cf1b tempest-ServersTestJSON-27047865 tempest-ServersTestJSON-27047865-project-member] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.089s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1838.396904] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 16.026s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1838.397101] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: fcfd0672-c583-4cb0-9e39-37b6e7cdb68e] During sync_power_state the instance has a pending task (deleting). Skip. [ 1838.397315] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "fcfd0672-c583-4cb0-9e39-37b6e7cdb68e" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1850.260797] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "3e8b50e4-64fb-4566-b730-9acf5591cbda" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1861.199401] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1861.199692] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1861.701984] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1862.224248] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1862.224508] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1862.225926] env[63593]: INFO nova.compute.claims [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1863.332449] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc6d88b2-1dc2-481f-8566-e2cd53f142ea {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1863.340122] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b211a03f-dd6a-4429-a491-f50b7979fada {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1863.369683] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47ba171d-1791-458b-9dfc-50636b23ba10 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1863.376922] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c42c281e-0a56-4d81-b965-797292a41528 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1863.389329] env[63593]: DEBUG nova.compute.provider_tree [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1863.892286] env[63593]: DEBUG nova.scheduler.client.report [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1864.396952] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.172s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1864.397594] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1864.902756] env[63593]: DEBUG nova.compute.utils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1864.904201] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1864.904367] env[63593]: DEBUG nova.network.neutron [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1864.939776] env[63593]: DEBUG nova.policy [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3bb930b4d1534f3ea3abde038aa60100', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3101ac6769714328b1b7e2c5f371bccc', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 1865.173675] env[63593]: DEBUG nova.network.neutron [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Successfully created port: 344ced4e-da8a-4f31-8c66-65dd2f76d867 {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1865.407139] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1866.417749] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1866.442694] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1866.442939] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1866.443104] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1866.443283] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1866.443426] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1866.443567] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1866.443769] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1866.443922] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1866.444094] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1866.444259] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1866.444424] env[63593]: DEBUG nova.virt.hardware [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1866.445310] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1fb557a4-3a90-403c-a85c-6e28de656906 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1866.452492] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ca9c48e-5378-4199-87a6-38c8e62244bd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1866.736170] env[63593]: DEBUG nova.compute.manager [req-e36499f4-18ad-4f59-b87a-36130e86c1ea req-b366dffe-3986-4dfc-a61f-c1b088f4de0b service nova] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Received event network-vif-plugged-344ced4e-da8a-4f31-8c66-65dd2f76d867 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1866.736463] env[63593]: DEBUG oslo_concurrency.lockutils [req-e36499f4-18ad-4f59-b87a-36130e86c1ea req-b366dffe-3986-4dfc-a61f-c1b088f4de0b service nova] Acquiring lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1866.736680] env[63593]: DEBUG oslo_concurrency.lockutils [req-e36499f4-18ad-4f59-b87a-36130e86c1ea req-b366dffe-3986-4dfc-a61f-c1b088f4de0b service nova] Lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1866.736850] env[63593]: DEBUG oslo_concurrency.lockutils [req-e36499f4-18ad-4f59-b87a-36130e86c1ea req-b366dffe-3986-4dfc-a61f-c1b088f4de0b service nova] Lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1866.737027] env[63593]: DEBUG nova.compute.manager [req-e36499f4-18ad-4f59-b87a-36130e86c1ea req-b366dffe-3986-4dfc-a61f-c1b088f4de0b service nova] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] No waiting events found dispatching network-vif-plugged-344ced4e-da8a-4f31-8c66-65dd2f76d867 {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1866.737196] env[63593]: WARNING nova.compute.manager [req-e36499f4-18ad-4f59-b87a-36130e86c1ea req-b366dffe-3986-4dfc-a61f-c1b088f4de0b service nova] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Received unexpected event network-vif-plugged-344ced4e-da8a-4f31-8c66-65dd2f76d867 for instance with vm_state building and task_state spawning. [ 1866.803685] env[63593]: DEBUG nova.network.neutron [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Successfully updated port: 344ced4e-da8a-4f31-8c66-65dd2f76d867 {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1867.305891] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "refresh_cache-053c6990-4a0f-4a9c-abfc-1b4b984f42cd" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1867.306088] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "refresh_cache-053c6990-4a0f-4a9c-abfc-1b4b984f42cd" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1867.306148] env[63593]: DEBUG nova.network.neutron [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1867.840369] env[63593]: DEBUG nova.network.neutron [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1867.968825] env[63593]: DEBUG nova.network.neutron [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Updating instance_info_cache with network_info: [{"id": "344ced4e-da8a-4f31-8c66-65dd2f76d867", "address": "fa:16:3e:8e:8a:06", "network": {"id": "aca8be2a-21d8-4636-8216-f46c0892ddc8", "bridge": "br-int", "label": "tempest-ServersTestJSON-848574930-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3101ac6769714328b1b7e2c5f371bccc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap344ced4e-da", "ovs_interfaceid": "344ced4e-da8a-4f31-8c66-65dd2f76d867", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1868.471924] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "refresh_cache-053c6990-4a0f-4a9c-abfc-1b4b984f42cd" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1868.472219] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Instance network_info: |[{"id": "344ced4e-da8a-4f31-8c66-65dd2f76d867", "address": "fa:16:3e:8e:8a:06", "network": {"id": "aca8be2a-21d8-4636-8216-f46c0892ddc8", "bridge": "br-int", "label": "tempest-ServersTestJSON-848574930-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3101ac6769714328b1b7e2c5f371bccc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap344ced4e-da", "ovs_interfaceid": "344ced4e-da8a-4f31-8c66-65dd2f76d867", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 1868.472640] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:8e:8a:06', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '7d377d75-3add-4a15-8691-74b2eb010924', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '344ced4e-da8a-4f31-8c66-65dd2f76d867', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1868.479966] env[63593]: DEBUG oslo.service.loopingcall [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1868.480176] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1868.480394] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-044e178e-9f05-49a7-b688-a34f90f41c4a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1868.500215] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1868.500215] env[63593]: value = "task-1368087" [ 1868.500215] env[63593]: _type = "Task" [ 1868.500215] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1868.507365] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368087, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1868.763062] env[63593]: DEBUG nova.compute.manager [req-d290497e-9da8-47b1-bfa4-e934e8c9dbc2 req-f9a98ffd-af89-448d-987b-2dee6ba19f15 service nova] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Received event network-changed-344ced4e-da8a-4f31-8c66-65dd2f76d867 {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1868.763271] env[63593]: DEBUG nova.compute.manager [req-d290497e-9da8-47b1-bfa4-e934e8c9dbc2 req-f9a98ffd-af89-448d-987b-2dee6ba19f15 service nova] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Refreshing instance network info cache due to event network-changed-344ced4e-da8a-4f31-8c66-65dd2f76d867. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1868.763488] env[63593]: DEBUG oslo_concurrency.lockutils [req-d290497e-9da8-47b1-bfa4-e934e8c9dbc2 req-f9a98ffd-af89-448d-987b-2dee6ba19f15 service nova] Acquiring lock "refresh_cache-053c6990-4a0f-4a9c-abfc-1b4b984f42cd" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1868.763627] env[63593]: DEBUG oslo_concurrency.lockutils [req-d290497e-9da8-47b1-bfa4-e934e8c9dbc2 req-f9a98ffd-af89-448d-987b-2dee6ba19f15 service nova] Acquired lock "refresh_cache-053c6990-4a0f-4a9c-abfc-1b4b984f42cd" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1868.763786] env[63593]: DEBUG nova.network.neutron [req-d290497e-9da8-47b1-bfa4-e934e8c9dbc2 req-f9a98ffd-af89-448d-987b-2dee6ba19f15 service nova] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Refreshing network info cache for port 344ced4e-da8a-4f31-8c66-65dd2f76d867 {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1869.009668] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368087, 'name': CreateVM_Task, 'duration_secs': 0.288157} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1869.010027] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1869.010491] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1869.010661] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1869.010970] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1869.011220] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-90cd04d1-7589-4786-a492-d6cafc95a4a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1869.015355] env[63593]: DEBUG oslo_vmware.api [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 1869.015355] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52d184c4-0b7d-0533-27a2-f6e4ce4e5ced" [ 1869.015355] env[63593]: _type = "Task" [ 1869.015355] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1869.022339] env[63593]: DEBUG oslo_vmware.api [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52d184c4-0b7d-0533-27a2-f6e4ce4e5ced, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1869.465554] env[63593]: DEBUG nova.network.neutron [req-d290497e-9da8-47b1-bfa4-e934e8c9dbc2 req-f9a98ffd-af89-448d-987b-2dee6ba19f15 service nova] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Updated VIF entry in instance network info cache for port 344ced4e-da8a-4f31-8c66-65dd2f76d867. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1869.465916] env[63593]: DEBUG nova.network.neutron [req-d290497e-9da8-47b1-bfa4-e934e8c9dbc2 req-f9a98ffd-af89-448d-987b-2dee6ba19f15 service nova] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Updating instance_info_cache with network_info: [{"id": "344ced4e-da8a-4f31-8c66-65dd2f76d867", "address": "fa:16:3e:8e:8a:06", "network": {"id": "aca8be2a-21d8-4636-8216-f46c0892ddc8", "bridge": "br-int", "label": "tempest-ServersTestJSON-848574930-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3101ac6769714328b1b7e2c5f371bccc", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap344ced4e-da", "ovs_interfaceid": "344ced4e-da8a-4f31-8c66-65dd2f76d867", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1869.525260] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1869.525482] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1869.525663] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1869.968382] env[63593]: DEBUG oslo_concurrency.lockutils [req-d290497e-9da8-47b1-bfa4-e934e8c9dbc2 req-f9a98ffd-af89-448d-987b-2dee6ba19f15 service nova] Releasing lock "refresh_cache-053c6990-4a0f-4a9c-abfc-1b4b984f42cd" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1878.940927] env[63593]: WARNING oslo_vmware.rw_handles [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1878.940927] env[63593]: ERROR oslo_vmware.rw_handles [ 1878.941625] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/741eaeb6-c741-41cc-814a-1680f89b1f04/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1878.943423] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1878.943698] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Copying Virtual Disk [datastore2] vmware_temp/741eaeb6-c741-41cc-814a-1680f89b1f04/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/741eaeb6-c741-41cc-814a-1680f89b1f04/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1878.944119] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-4814d527-ce05-4388-87c5-c041e7b118ca {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1878.951871] env[63593]: DEBUG oslo_vmware.api [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 1878.951871] env[63593]: value = "task-1368088" [ 1878.951871] env[63593]: _type = "Task" [ 1878.951871] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1878.959797] env[63593]: DEBUG oslo_vmware.api [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': task-1368088, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1879.461741] env[63593]: DEBUG oslo_vmware.exceptions [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1879.462049] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1879.462624] env[63593]: ERROR nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1879.462624] env[63593]: Faults: ['InvalidArgument'] [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Traceback (most recent call last): [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] yield resources [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] self.driver.spawn(context, instance, image_meta, [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] self._fetch_image_if_missing(context, vi) [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] image_cache(vi, tmp_image_ds_loc) [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] vm_util.copy_virtual_disk( [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] session._wait_for_task(vmdk_copy_task) [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] return self.wait_for_task(task_ref) [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] return evt.wait() [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] result = hub.switch() [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] return self.greenlet.switch() [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] self.f(*self.args, **self.kw) [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] raise exceptions.translate_fault(task_info.error) [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Faults: ['InvalidArgument'] [ 1879.462624] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] [ 1879.463831] env[63593]: INFO nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Terminating instance [ 1879.464545] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1879.464708] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1879.464938] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-957349a7-95ce-4ec4-b12d-c08021606cb6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.467256] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1879.467444] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1879.468179] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0dad802-3147-43cf-a618-a8207dd8dfa8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.474408] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1879.474608] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-847251db-06c8-4e73-94a2-7f8da1c45b18 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.476665] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1879.476830] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1879.477751] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-dc2486e0-8180-44fc-a282-db23f2a40e17 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.483793] env[63593]: DEBUG oslo_vmware.api [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 1879.483793] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5259e818-97ab-6c97-9ade-1c7c21eff0df" [ 1879.483793] env[63593]: _type = "Task" [ 1879.483793] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1879.497891] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1879.498140] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] vmware_temp/cfd48e2c-abbd-4dde-976b-e2dd7d950699/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1879.498344] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-3d952ffb-2ece-4edb-a170-4cb095bdee2e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.517632] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] vmware_temp/cfd48e2c-abbd-4dde-976b-e2dd7d950699/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1879.517861] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Fetch image to [datastore2] vmware_temp/cfd48e2c-abbd-4dde-976b-e2dd7d950699/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1879.518076] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/cfd48e2c-abbd-4dde-976b-e2dd7d950699/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1879.518780] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5916b8f0-70f2-4393-92f7-b785133b1171 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.525302] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40b83a28-45da-4e5c-93b1-254300a75814 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.534095] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dcc9cd3-a95d-40fb-b770-fc43cc33716a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.539618] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1879.539846] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1879.540031] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Deleting the datastore file [datastore2] 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1879.563388] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-70763fc6-6fb6-4296-b45e-cfa6c761b93b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.565757] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfff3ba4-4328-47ce-9670-15693e365f51 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.572500] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-a4f15dcd-d050-479f-8c7d-b95cb434bcd4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.574125] env[63593]: DEBUG oslo_vmware.api [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 1879.574125] env[63593]: value = "task-1368090" [ 1879.574125] env[63593]: _type = "Task" [ 1879.574125] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1879.581978] env[63593]: DEBUG oslo_vmware.api [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': task-1368090, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1879.597096] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1879.641632] env[63593]: DEBUG oslo_vmware.rw_handles [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/cfd48e2c-abbd-4dde-976b-e2dd7d950699/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1879.702410] env[63593]: DEBUG oslo_vmware.rw_handles [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1879.702597] env[63593]: DEBUG oslo_vmware.rw_handles [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/cfd48e2c-abbd-4dde-976b-e2dd7d950699/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1880.083945] env[63593]: DEBUG oslo_vmware.api [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': task-1368090, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.064766} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1880.084356] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1880.084406] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1880.084546] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1880.084723] env[63593]: INFO nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1880.086814] env[63593]: DEBUG nova.compute.claims [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1880.086980] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1880.087204] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1880.710409] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-406ab60d-93dd-4267-894b-9118c8771710 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.717415] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dbb98db5-d1cb-4d30-a947-463e8bc36b6e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.747236] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-160ed111-71b0-4d83-9ea2-dce6f3c36fdb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.754037] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0917472b-e906-48e7-86f5-e3c4d830419b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1880.766776] env[63593]: DEBUG nova.compute.provider_tree [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1881.270497] env[63593]: DEBUG nova.scheduler.client.report [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1881.775676] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.688s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1881.776268] env[63593]: ERROR nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1881.776268] env[63593]: Faults: ['InvalidArgument'] [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Traceback (most recent call last): [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] self.driver.spawn(context, instance, image_meta, [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] self._fetch_image_if_missing(context, vi) [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] image_cache(vi, tmp_image_ds_loc) [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] vm_util.copy_virtual_disk( [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] session._wait_for_task(vmdk_copy_task) [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] return self.wait_for_task(task_ref) [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] return evt.wait() [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] result = hub.switch() [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] return self.greenlet.switch() [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] self.f(*self.args, **self.kw) [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] raise exceptions.translate_fault(task_info.error) [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Faults: ['InvalidArgument'] [ 1881.776268] env[63593]: ERROR nova.compute.manager [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] [ 1881.777235] env[63593]: DEBUG nova.compute.utils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1881.778833] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Build of instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf was re-scheduled: A specified parameter was not correct: fileType [ 1881.778833] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1881.779213] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1881.779382] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1881.779548] env[63593]: DEBUG nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1881.779704] env[63593]: DEBUG nova.network.neutron [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1882.533164] env[63593]: DEBUG nova.network.neutron [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1883.036178] env[63593]: INFO nova.compute.manager [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Took 1.26 seconds to deallocate network for instance. [ 1884.070486] env[63593]: INFO nova.scheduler.client.report [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Deleted allocations for instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf [ 1884.281741] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1884.281991] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1884.282103] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1884.578453] env[63593]: DEBUG oslo_concurrency.lockutils [None req-25393c45-3295-4c49-a79e-456948c5c256 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 666.977s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1884.578715] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 471.341s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1884.578953] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1884.579189] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1884.579355] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1884.582886] env[63593]: INFO nova.compute.manager [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Terminating instance [ 1884.584565] env[63593]: DEBUG nova.compute.manager [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1884.584753] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1884.585087] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a4f4af19-83de-49f2-82ce-ca1ca011626c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1884.593125] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7edef630-817b-483c-9934-d55fce538da9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1884.618186] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf could not be found. [ 1884.618396] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1884.618574] env[63593]: INFO nova.compute.manager [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1884.618815] env[63593]: DEBUG oslo.service.loopingcall [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1884.619041] env[63593]: DEBUG nova.compute.manager [-] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1884.619136] env[63593]: DEBUG nova.network.neutron [-] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1884.787207] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1884.787409] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1884.787527] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1884.787637] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1884.787759] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1884.787877] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1884.787996] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1884.788130] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1884.788252] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1884.788446] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1884.788608] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1884.788733] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1885.141840] env[63593]: DEBUG nova.network.neutron [-] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1885.645634] env[63593]: INFO nova.compute.manager [-] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] Took 1.03 seconds to deallocate network for instance. [ 1886.012642] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1886.012807] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1886.012986] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1886.671026] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4f2df6d3-c6b1-42d7-b9b4-e908cee19ddc tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.092s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1886.671934] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 64.300s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1886.672495] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 2f7b32d3-2b3a-4400-ab45-dc9c097af2bf] During sync_power_state the instance has a pending task (deleting). Skip. [ 1886.672682] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "2f7b32d3-2b3a-4400-ab45-dc9c097af2bf" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1887.012941] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1890.008502] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1892.007688] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1895.012617] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1895.516051] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1895.516803] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1895.516803] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1895.516803] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1895.517364] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f040e2ee-2a5c-4dda-a21f-04a7192f76a2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1895.525813] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c92eccc-d15c-494c-9362-abc017e5a032 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1895.539060] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-972b85a6-cc40-443b-b332-4cff1fb9b8d8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1895.545234] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de38dd16-f998-4c02-913c-b255529aa606 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1895.574599] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181446MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1895.574785] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1895.574952] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1896.604375] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance a1b46566-4db2-4086-906a-7ba32b9a2813 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1896.604680] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1896.604680] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1896.604810] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1896.604868] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1896.604975] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 3e8b50e4-64fb-4566-b730-9acf5591cbda actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1896.605104] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 606babde-3951-4b7a-b4bd-f7c22ab09195 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1896.605217] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1896.605407] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 8 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1896.605541] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=2048MB phys_disk=200GB used_disk=8GB total_vcpus=48 used_vcpus=8 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1896.698566] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75d9bca2-0446-4122-9605-e528e1a526fd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1896.705541] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-690d51fc-05e2-418e-aa4c-354575681cc9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1896.734987] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0857eac-928b-4fbe-8f39-c7add618a19f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1896.741815] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7576c82e-d90c-47df-99da-b1bd02688c29 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1896.754729] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1897.257677] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1897.763087] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1897.763430] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.188s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1925.679206] env[63593]: WARNING oslo_vmware.rw_handles [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1925.679206] env[63593]: ERROR oslo_vmware.rw_handles [ 1925.679944] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/cfd48e2c-abbd-4dde-976b-e2dd7d950699/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1925.681864] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1925.682158] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Copying Virtual Disk [datastore2] vmware_temp/cfd48e2c-abbd-4dde-976b-e2dd7d950699/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/cfd48e2c-abbd-4dde-976b-e2dd7d950699/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1925.682469] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-f5465995-e58d-4db2-be72-a71b54d356fd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1925.689906] env[63593]: DEBUG oslo_vmware.api [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 1925.689906] env[63593]: value = "task-1368091" [ 1925.689906] env[63593]: _type = "Task" [ 1925.689906] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1925.697420] env[63593]: DEBUG oslo_vmware.api [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368091, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1926.201248] env[63593]: DEBUG oslo_vmware.exceptions [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1926.201248] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1926.201479] env[63593]: ERROR nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1926.201479] env[63593]: Faults: ['InvalidArgument'] [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Traceback (most recent call last): [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] yield resources [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] self.driver.spawn(context, instance, image_meta, [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] self._fetch_image_if_missing(context, vi) [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] image_cache(vi, tmp_image_ds_loc) [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] vm_util.copy_virtual_disk( [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] session._wait_for_task(vmdk_copy_task) [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] return self.wait_for_task(task_ref) [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] return evt.wait() [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] result = hub.switch() [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] return self.greenlet.switch() [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] self.f(*self.args, **self.kw) [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] raise exceptions.translate_fault(task_info.error) [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Faults: ['InvalidArgument'] [ 1926.201479] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] [ 1926.202492] env[63593]: INFO nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Terminating instance [ 1926.203296] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1926.204134] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1926.204425] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-732aa924-1404-4658-9d6a-419f43150bc5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.207439] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1926.207638] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1926.208400] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07fc952e-606d-4ce4-9c5f-bf391169903b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.211786] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1926.211955] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1926.212921] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2e469167-8a98-4bf8-8a6a-5644ba2e3884 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.216589] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1926.217079] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0c9846c8-e036-4bb4-b671-fe41d6dd7061 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.219185] env[63593]: DEBUG oslo_vmware.api [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for the task: (returnval){ [ 1926.219185] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52d5d374-3afb-cc21-8656-f82f308b33db" [ 1926.219185] env[63593]: _type = "Task" [ 1926.219185] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1926.227926] env[63593]: DEBUG oslo_vmware.api [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52d5d374-3afb-cc21-8656-f82f308b33db, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1926.289799] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1926.290044] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1926.290216] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleting the datastore file [datastore2] a1b46566-4db2-4086-906a-7ba32b9a2813 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1926.290556] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-725068b6-fee4-4e09-b063-abc1f0dcc7cb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.296430] env[63593]: DEBUG oslo_vmware.api [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 1926.296430] env[63593]: value = "task-1368093" [ 1926.296430] env[63593]: _type = "Task" [ 1926.296430] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1926.303770] env[63593]: DEBUG oslo_vmware.api [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368093, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1926.728792] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1926.729169] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Creating directory with path [datastore2] vmware_temp/4ab434db-3d1f-4b6d-a679-c4ccf76c8fdb/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1926.729254] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-59c010ac-0c6e-4b83-b82e-84675bd261e4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.739793] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Created directory with path [datastore2] vmware_temp/4ab434db-3d1f-4b6d-a679-c4ccf76c8fdb/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1926.739997] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Fetch image to [datastore2] vmware_temp/4ab434db-3d1f-4b6d-a679-c4ccf76c8fdb/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1926.740187] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/4ab434db-3d1f-4b6d-a679-c4ccf76c8fdb/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1926.740883] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da3af64a-0ffa-4c21-b3d0-f75324a0f648 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.747407] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e07b56fb-728c-4fce-968f-ea24e8220863 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.756077] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69d68d45-75ef-4ac4-acfc-f3140e310992 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.787014] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-493b483a-5de1-4eab-9316-19de4991b1c5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.792903] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-8801787c-8746-4121-a5c9-321170d83247 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1926.803780] env[63593]: DEBUG oslo_vmware.api [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368093, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.063582} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1926.804035] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1926.804227] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1926.804405] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1926.804585] env[63593]: INFO nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1926.806673] env[63593]: DEBUG nova.compute.claims [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1926.806840] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1926.807065] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1926.811462] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1926.959025] env[63593]: DEBUG oslo_vmware.rw_handles [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4ab434db-3d1f-4b6d-a679-c4ccf76c8fdb/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1927.019739] env[63593]: DEBUG oslo_vmware.rw_handles [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1927.019917] env[63593]: DEBUG oslo_vmware.rw_handles [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4ab434db-3d1f-4b6d-a679-c4ccf76c8fdb/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1927.406354] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-330d60ad-d109-4848-bfa6-461dfe13c492 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1927.415082] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2cc9233-7c7a-4b91-a1a0-1511b4301f7e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1927.443562] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfcfa06f-659f-4e56-a598-826b21405dcd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1927.449964] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e088d9ca-f17c-475e-982b-a98ae67d5ef1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1927.462508] env[63593]: DEBUG nova.compute.provider_tree [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1927.965274] env[63593]: DEBUG nova.scheduler.client.report [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1928.470735] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.663s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1928.471308] env[63593]: ERROR nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1928.471308] env[63593]: Faults: ['InvalidArgument'] [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Traceback (most recent call last): [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] self.driver.spawn(context, instance, image_meta, [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] self._fetch_image_if_missing(context, vi) [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] image_cache(vi, tmp_image_ds_loc) [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] vm_util.copy_virtual_disk( [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] session._wait_for_task(vmdk_copy_task) [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] return self.wait_for_task(task_ref) [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] return evt.wait() [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] result = hub.switch() [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] return self.greenlet.switch() [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] self.f(*self.args, **self.kw) [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] raise exceptions.translate_fault(task_info.error) [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Faults: ['InvalidArgument'] [ 1928.471308] env[63593]: ERROR nova.compute.manager [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] [ 1928.472182] env[63593]: DEBUG nova.compute.utils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1928.473784] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Build of instance a1b46566-4db2-4086-906a-7ba32b9a2813 was re-scheduled: A specified parameter was not correct: fileType [ 1928.473784] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1928.474157] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1928.474328] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1928.474498] env[63593]: DEBUG nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1928.474656] env[63593]: DEBUG nova.network.neutron [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1929.247577] env[63593]: DEBUG nova.network.neutron [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1929.751053] env[63593]: INFO nova.compute.manager [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Took 1.28 seconds to deallocate network for instance. [ 1930.784279] env[63593]: INFO nova.scheduler.client.report [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted allocations for instance a1b46566-4db2-4086-906a-7ba32b9a2813 [ 1931.294778] env[63593]: DEBUG oslo_concurrency.lockutils [None req-873748ad-e697-4319-8123-43f48ddd4e86 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 669.906s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1931.295086] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 473.839s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1931.295331] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "a1b46566-4db2-4086-906a-7ba32b9a2813-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1931.295545] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1931.295734] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1931.297951] env[63593]: INFO nova.compute.manager [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Terminating instance [ 1931.299716] env[63593]: DEBUG nova.compute.manager [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1931.299904] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1931.300180] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bb7f4191-5447-4dee-a0b8-786fd988c8a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1931.309278] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b45c10e-aa3d-4bfc-89d9-5ce69fae9c95 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1931.335108] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a1b46566-4db2-4086-906a-7ba32b9a2813 could not be found. [ 1931.335316] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1931.335500] env[63593]: INFO nova.compute.manager [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1931.335746] env[63593]: DEBUG oslo.service.loopingcall [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1931.335964] env[63593]: DEBUG nova.compute.manager [-] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1931.336071] env[63593]: DEBUG nova.network.neutron [-] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1931.907430] env[63593]: DEBUG nova.network.neutron [-] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1932.410074] env[63593]: INFO nova.compute.manager [-] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] Took 1.07 seconds to deallocate network for instance. [ 1933.435068] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d9d4ba42-a334-49f9-bbc5-077defada2a4 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.139s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1933.435454] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 111.063s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1933.435581] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: a1b46566-4db2-4086-906a-7ba32b9a2813] During sync_power_state the instance has a pending task (deleting). Skip. [ 1933.435754] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "a1b46566-4db2-4086-906a-7ba32b9a2813" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1944.763630] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1944.764041] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1944.764041] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1945.268427] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1945.268599] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1945.268731] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1945.268856] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1945.268978] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1945.269117] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1945.269239] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 1945.269358] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1945.269558] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1945.269728] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1945.269849] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1946.012396] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1946.012755] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1947.012601] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1948.013445] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1952.009103] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1955.015014] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1955.521246] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1955.521481] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1955.521651] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1955.521835] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1955.522732] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8d1609f-0fdf-4d93-9135-0d323b22b314 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.531345] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b69cd8a-4ffb-43df-9bcd-db5eb936aece {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.545612] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22c748cf-0c1d-48d1-a65d-6fe7382cb9b4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.552380] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f12bf918-deb2-43d0-8239-655c023ddcd7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.580403] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181457MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1955.580546] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1955.580830] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1956.610977] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1956.611302] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1956.611302] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1956.611438] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1956.611513] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 3e8b50e4-64fb-4566-b730-9acf5591cbda actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1956.611599] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 606babde-3951-4b7a-b4bd-f7c22ab09195 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1956.611714] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1956.611921] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 7 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1956.612072] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1856MB phys_disk=200GB used_disk=7GB total_vcpus=48 used_vcpus=7 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1956.695917] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77400e54-0a83-48b4-b2e6-d5d02a47e3c4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.702903] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80eec67d-11bd-4fb3-81cd-aa883624f003 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.732393] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-262c6287-bcfc-44d2-a4ef-87ff43887b5b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.738853] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67465eef-24ac-4c0a-a0b7-9c6c63c7b5ee {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1956.751067] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1957.254367] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1957.759964] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1957.760285] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.179s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1972.839381] env[63593]: WARNING oslo_vmware.rw_handles [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1972.839381] env[63593]: ERROR oslo_vmware.rw_handles [ 1972.840372] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/4ab434db-3d1f-4b6d-a679-c4ccf76c8fdb/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1972.841958] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1972.842207] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Copying Virtual Disk [datastore2] vmware_temp/4ab434db-3d1f-4b6d-a679-c4ccf76c8fdb/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/4ab434db-3d1f-4b6d-a679-c4ccf76c8fdb/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1972.842488] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-0b6179ee-a9b4-404d-bb82-a9d4a8ae4270 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1972.851618] env[63593]: DEBUG oslo_vmware.api [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for the task: (returnval){ [ 1972.851618] env[63593]: value = "task-1368094" [ 1972.851618] env[63593]: _type = "Task" [ 1972.851618] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1972.859211] env[63593]: DEBUG oslo_vmware.api [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': task-1368094, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1973.363242] env[63593]: DEBUG oslo_vmware.exceptions [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1973.363242] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1973.363242] env[63593]: ERROR nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1973.363242] env[63593]: Faults: ['InvalidArgument'] [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Traceback (most recent call last): [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] yield resources [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] self.driver.spawn(context, instance, image_meta, [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] self._fetch_image_if_missing(context, vi) [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] image_cache(vi, tmp_image_ds_loc) [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] vm_util.copy_virtual_disk( [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] session._wait_for_task(vmdk_copy_task) [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] return self.wait_for_task(task_ref) [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] return evt.wait() [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] result = hub.switch() [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] return self.greenlet.switch() [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] self.f(*self.args, **self.kw) [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] raise exceptions.translate_fault(task_info.error) [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Faults: ['InvalidArgument'] [ 1973.363242] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] [ 1973.364451] env[63593]: INFO nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Terminating instance [ 1973.364979] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1973.365200] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1973.365429] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8be8813c-055c-4d9e-975a-c847aa79f50a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.367509] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1973.367703] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1973.368404] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c39c9c3e-ecb8-4dbb-9b0e-eb74358fbd9f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.374982] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1973.375194] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-8f1d80fc-9906-477a-9bed-af426828be0f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.377101] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1973.377275] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1973.378186] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-18bd34b3-34f7-4d1e-8d67-9d97886746e2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.382607] env[63593]: DEBUG oslo_vmware.api [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Waiting for the task: (returnval){ [ 1973.382607] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]527dfe58-1f27-3cd5-cd0b-744d9e469202" [ 1973.382607] env[63593]: _type = "Task" [ 1973.382607] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1973.395970] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1973.396204] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Creating directory with path [datastore2] vmware_temp/d7237143-f00d-4394-8de4-51b1997c21c4/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1973.396405] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-977962d0-ef1f-4a98-b3e5-69add3b9b71c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.416523] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Created directory with path [datastore2] vmware_temp/d7237143-f00d-4394-8de4-51b1997c21c4/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1973.416704] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Fetch image to [datastore2] vmware_temp/d7237143-f00d-4394-8de4-51b1997c21c4/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1973.416870] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/d7237143-f00d-4394-8de4-51b1997c21c4/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1973.417585] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d9931a5-fe43-4875-b39d-101a07742e72 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.424992] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25746128-7396-434c-853f-cef28e21ec7d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.433729] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eddf2b72-a78c-4fcf-8e9a-28ba8e461f9f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.464205] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6416c763-5b05-4eff-981e-783a1c9f5bc3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.466511] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1973.466700] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1973.466866] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Deleting the datastore file [datastore2] abaeda1e-4ad8-4553-9fb0-f94e2cd98059 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1973.467094] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ad967ca6-64e9-4ab9-9b6e-ba5a3bba79f4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.471416] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-e4ae46d2-e425-4c23-9f62-7a9ddc709bb2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1973.474049] env[63593]: DEBUG oslo_vmware.api [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for the task: (returnval){ [ 1973.474049] env[63593]: value = "task-1368096" [ 1973.474049] env[63593]: _type = "Task" [ 1973.474049] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1973.480952] env[63593]: DEBUG oslo_vmware.api [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': task-1368096, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1973.496255] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1973.546920] env[63593]: DEBUG oslo_vmware.rw_handles [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d7237143-f00d-4394-8de4-51b1997c21c4/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1973.607393] env[63593]: DEBUG oslo_vmware.rw_handles [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1973.607590] env[63593]: DEBUG oslo_vmware.rw_handles [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d7237143-f00d-4394-8de4-51b1997c21c4/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1973.984158] env[63593]: DEBUG oslo_vmware.api [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Task: {'id': task-1368096, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.067666} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1973.984476] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1973.984565] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1973.984732] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1973.984896] env[63593]: INFO nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1973.986992] env[63593]: DEBUG nova.compute.claims [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1973.987172] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1973.987431] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1974.571349] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97238990-9f8f-40a7-9e72-95333f96ef98 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1974.578583] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-63f77b3b-b677-4eae-bfd9-720c1651db2b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1974.608471] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e565141-7f2a-4d58-bab8-739256b8e79a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1974.615560] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad3ee677-3664-4bef-885d-70e0a6643579 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1974.628177] env[63593]: DEBUG nova.compute.provider_tree [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1975.131714] env[63593]: DEBUG nova.scheduler.client.report [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1975.636782] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.649s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1975.637394] env[63593]: ERROR nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1975.637394] env[63593]: Faults: ['InvalidArgument'] [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Traceback (most recent call last): [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] self.driver.spawn(context, instance, image_meta, [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] self._fetch_image_if_missing(context, vi) [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] image_cache(vi, tmp_image_ds_loc) [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] vm_util.copy_virtual_disk( [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] session._wait_for_task(vmdk_copy_task) [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] return self.wait_for_task(task_ref) [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] return evt.wait() [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] result = hub.switch() [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] return self.greenlet.switch() [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] self.f(*self.args, **self.kw) [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] raise exceptions.translate_fault(task_info.error) [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Faults: ['InvalidArgument'] [ 1975.637394] env[63593]: ERROR nova.compute.manager [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] [ 1975.638498] env[63593]: DEBUG nova.compute.utils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1975.640098] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Build of instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 was re-scheduled: A specified parameter was not correct: fileType [ 1975.640098] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1975.640455] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1975.640624] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1975.640791] env[63593]: DEBUG nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1975.640985] env[63593]: DEBUG nova.network.neutron [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1976.429189] env[63593]: DEBUG nova.network.neutron [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1976.932262] env[63593]: INFO nova.compute.manager [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Took 1.29 seconds to deallocate network for instance. [ 1977.968515] env[63593]: INFO nova.scheduler.client.report [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Deleted allocations for instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 [ 1978.476679] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e6aeccd3-c5ef-46f6-8245-b32cd527ae9b tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 558.274s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1978.476959] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 362.075s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1978.477259] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Acquiring lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1978.477481] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1978.477646] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1978.480409] env[63593]: INFO nova.compute.manager [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Terminating instance [ 1978.482190] env[63593]: DEBUG nova.compute.manager [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1978.482387] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1978.482645] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-df7c0422-3938-4237-a123-9fa71b2f9811 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1978.491309] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c6e8191-a199-4114-83e9-922b49c45dd5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1978.516400] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance abaeda1e-4ad8-4553-9fb0-f94e2cd98059 could not be found. [ 1978.516592] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1978.516763] env[63593]: INFO nova.compute.manager [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1978.516992] env[63593]: DEBUG oslo.service.loopingcall [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1978.517222] env[63593]: DEBUG nova.compute.manager [-] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1978.517315] env[63593]: DEBUG nova.network.neutron [-] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1979.034327] env[63593]: DEBUG nova.network.neutron [-] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1979.537254] env[63593]: INFO nova.compute.manager [-] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] Took 1.02 seconds to deallocate network for instance. [ 1980.561010] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a227861d-6de4-4ea6-a510-8bd2d88e302f tempest-ServerDiskConfigTestJSON-856453587 tempest-ServerDiskConfigTestJSON-856453587-project-member] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.084s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1980.562134] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 158.190s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1980.562134] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: abaeda1e-4ad8-4553-9fb0-f94e2cd98059] During sync_power_state the instance has a pending task (deleting). Skip. [ 1980.562275] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "abaeda1e-4ad8-4553-9fb0-f94e2cd98059" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2006.758469] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2006.758887] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 2006.758887] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 2007.263443] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2007.263638] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2007.263778] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2007.263903] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2007.264037] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2007.264160] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2007.264281] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 2007.264481] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2007.264635] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2007.264781] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2007.264926] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2007.265057] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 2009.013225] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2009.013538] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2012.008018] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2012.246567] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquiring lock "606babde-3951-4b7a-b4bd-f7c22ab09195" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2014.513595] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2015.013068] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2015.516351] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2015.516751] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2015.516816] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2015.516941] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2015.517876] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bf5c346-1b93-44b8-bbd4-1c83aeb8a039 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2015.526431] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-751934de-aeba-4f9b-97c1-237ea29efad2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2015.539567] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f34b660-b850-4554-b779-5dc6fed11ff8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2015.545301] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd3bdb5c-e7de-496d-9332-761efff0949a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2015.572595] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181477MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2015.572745] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2015.572930] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2016.600837] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 0484e878-6799-4030-9dcc-d62d32317744 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2016.601139] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2016.601139] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2016.601229] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 3e8b50e4-64fb-4566-b730-9acf5591cbda actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2016.601386] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 606babde-3951-4b7a-b4bd-f7c22ab09195 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2016.601498] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2016.601680] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2016.601815] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2016.676239] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1741405-63e9-4654-b015-ef67c6069a72 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.683711] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a077838d-d305-47f5-9893-549d57655b57 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.712315] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d884fdf7-c619-42a7-85b5-21569fcecb3e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.720112] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7087b5a-3189-482e-80ac-33db95c405b0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2016.732697] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2017.235646] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2017.741487] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2017.741890] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.169s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2018.994445] env[63593]: WARNING oslo_vmware.rw_handles [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2018.994445] env[63593]: ERROR oslo_vmware.rw_handles [ 2018.995068] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/d7237143-f00d-4394-8de4-51b1997c21c4/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2018.996937] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2018.997193] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Copying Virtual Disk [datastore2] vmware_temp/d7237143-f00d-4394-8de4-51b1997c21c4/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/d7237143-f00d-4394-8de4-51b1997c21c4/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2018.997464] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-fb1622a7-72ba-4f02-8384-5d81443ec886 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.005661] env[63593]: DEBUG oslo_vmware.api [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Waiting for the task: (returnval){ [ 2019.005661] env[63593]: value = "task-1368097" [ 2019.005661] env[63593]: _type = "Task" [ 2019.005661] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2019.013281] env[63593]: DEBUG oslo_vmware.api [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Task: {'id': task-1368097, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2019.516673] env[63593]: DEBUG oslo_vmware.exceptions [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2019.516957] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2019.517513] env[63593]: ERROR nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2019.517513] env[63593]: Faults: ['InvalidArgument'] [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] Traceback (most recent call last): [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] yield resources [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] self.driver.spawn(context, instance, image_meta, [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] self._fetch_image_if_missing(context, vi) [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] image_cache(vi, tmp_image_ds_loc) [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] vm_util.copy_virtual_disk( [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] session._wait_for_task(vmdk_copy_task) [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] return self.wait_for_task(task_ref) [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] return evt.wait() [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] result = hub.switch() [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] return self.greenlet.switch() [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] self.f(*self.args, **self.kw) [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] raise exceptions.translate_fault(task_info.error) [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] Faults: ['InvalidArgument'] [ 2019.517513] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] [ 2019.518566] env[63593]: INFO nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Terminating instance [ 2019.519326] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2019.519545] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2019.519879] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-92e152ba-ba75-41e8-9147-83a2a2211f93 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.522036] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2019.522036] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2019.522717] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49839d78-f390-4301-9ac5-2ed7758d34f0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.529042] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2019.529244] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-7646cb9b-affd-4498-bbcc-5340dae7231c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.531291] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2019.531490] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2019.532383] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-16a3e2ab-30bd-4ca4-8d40-e8e80a3837a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.536690] env[63593]: DEBUG oslo_vmware.api [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 2019.536690] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52b5d5f9-e9d9-3422-a3fe-dce51b3d2548" [ 2019.536690] env[63593]: _type = "Task" [ 2019.536690] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2019.543576] env[63593]: DEBUG oslo_vmware.api [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52b5d5f9-e9d9-3422-a3fe-dce51b3d2548, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2019.597985] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2019.598232] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2019.598410] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Deleting the datastore file [datastore2] 0484e878-6799-4030-9dcc-d62d32317744 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2019.598659] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-27ee95e5-9f08-483a-a461-0d6407cbeeb6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2019.605131] env[63593]: DEBUG oslo_vmware.api [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Waiting for the task: (returnval){ [ 2019.605131] env[63593]: value = "task-1368099" [ 2019.605131] env[63593]: _type = "Task" [ 2019.605131] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2019.612345] env[63593]: DEBUG oslo_vmware.api [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Task: {'id': task-1368099, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2020.046235] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2020.046569] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating directory with path [datastore2] vmware_temp/2e8f7777-c1a3-4520-a441-cec63753ae61/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2020.046789] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-2f590c06-40b6-446e-bb74-d6c37e90c5b2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.058465] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Created directory with path [datastore2] vmware_temp/2e8f7777-c1a3-4520-a441-cec63753ae61/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2020.058694] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Fetch image to [datastore2] vmware_temp/2e8f7777-c1a3-4520-a441-cec63753ae61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2020.058876] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/2e8f7777-c1a3-4520-a441-cec63753ae61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2020.059684] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48c639e5-415c-4ef4-8157-1e5627afef9e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.066137] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2251c3f-3337-4436-822f-0f9aa2802a74 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.074872] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb58a224-ecfa-44e2-a61a-845847d835f8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.104711] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17ea89d2-b116-4e22-bcd2-098103b717b6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.115761] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4e7d205e-ab10-46a3-beff-05aa6d5d3f2d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.117353] env[63593]: DEBUG oslo_vmware.api [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Task: {'id': task-1368099, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.075878} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2020.117591] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2020.117766] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2020.117931] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2020.118106] env[63593]: INFO nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2020.120160] env[63593]: DEBUG nova.compute.claims [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 2020.120350] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2020.120567] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2020.140521] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2020.194362] env[63593]: DEBUG oslo_vmware.rw_handles [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/2e8f7777-c1a3-4520-a441-cec63753ae61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2020.256282] env[63593]: DEBUG oslo_vmware.rw_handles [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2020.256483] env[63593]: DEBUG oslo_vmware.rw_handles [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/2e8f7777-c1a3-4520-a441-cec63753ae61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2020.696996] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e079791f-3d27-44ff-9bca-8d5f6825ba2b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.704311] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a440c9c0-3287-44d5-84b2-ab94e47e536a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.732891] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40ee21ee-0ed6-476e-86de-b64c9818be31 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.739137] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c2f2187-f7b8-4b5f-a524-8596519589e4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2020.751491] env[63593]: DEBUG nova.compute.provider_tree [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2021.254571] env[63593]: DEBUG nova.scheduler.client.report [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2021.759960] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.639s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2021.760543] env[63593]: ERROR nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2021.760543] env[63593]: Faults: ['InvalidArgument'] [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] Traceback (most recent call last): [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] self.driver.spawn(context, instance, image_meta, [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] self._fetch_image_if_missing(context, vi) [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] image_cache(vi, tmp_image_ds_loc) [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] vm_util.copy_virtual_disk( [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] session._wait_for_task(vmdk_copy_task) [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] return self.wait_for_task(task_ref) [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] return evt.wait() [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] result = hub.switch() [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] return self.greenlet.switch() [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] self.f(*self.args, **self.kw) [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] raise exceptions.translate_fault(task_info.error) [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] Faults: ['InvalidArgument'] [ 2021.760543] env[63593]: ERROR nova.compute.manager [instance: 0484e878-6799-4030-9dcc-d62d32317744] [ 2021.761549] env[63593]: DEBUG nova.compute.utils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2021.763143] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Build of instance 0484e878-6799-4030-9dcc-d62d32317744 was re-scheduled: A specified parameter was not correct: fileType [ 2021.763143] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 2021.763513] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 2021.763682] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 2021.763933] env[63593]: DEBUG nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2021.764178] env[63593]: DEBUG nova.network.neutron [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2022.493896] env[63593]: DEBUG nova.network.neutron [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2022.997246] env[63593]: INFO nova.compute.manager [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Took 1.23 seconds to deallocate network for instance. [ 2024.032331] env[63593]: INFO nova.scheduler.client.report [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Deleted allocations for instance 0484e878-6799-4030-9dcc-d62d32317744 [ 2024.540673] env[63593]: DEBUG oslo_concurrency.lockutils [None req-4eb32c70-34ed-43f5-a7f9-ef0e39039d43 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "0484e878-6799-4030-9dcc-d62d32317744" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 594.553s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2024.541057] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "0484e878-6799-4030-9dcc-d62d32317744" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 398.411s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2024.541312] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Acquiring lock "0484e878-6799-4030-9dcc-d62d32317744-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2024.541552] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "0484e878-6799-4030-9dcc-d62d32317744-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2024.541727] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "0484e878-6799-4030-9dcc-d62d32317744-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2024.544814] env[63593]: INFO nova.compute.manager [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Terminating instance [ 2024.546710] env[63593]: DEBUG nova.compute.manager [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2024.546899] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2024.547187] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9cfcd0ed-1c48-4d4f-b6cc-b4da0eb31fea {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2024.555928] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-242df852-e87d-4d74-8b04-d483230dee57 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2024.583578] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0484e878-6799-4030-9dcc-d62d32317744 could not be found. [ 2024.583771] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2024.583947] env[63593]: INFO nova.compute.manager [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2024.584201] env[63593]: DEBUG oslo.service.loopingcall [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2024.584415] env[63593]: DEBUG nova.compute.manager [-] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2024.584509] env[63593]: DEBUG nova.network.neutron [-] [instance: 0484e878-6799-4030-9dcc-d62d32317744] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2025.101350] env[63593]: DEBUG nova.network.neutron [-] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2025.604522] env[63593]: INFO nova.compute.manager [-] [instance: 0484e878-6799-4030-9dcc-d62d32317744] Took 1.02 seconds to deallocate network for instance. [ 2026.630585] env[63593]: DEBUG oslo_concurrency.lockutils [None req-f32a595f-e75a-49fe-bd94-cbba181aeeb2 tempest-ServersNegativeTestJSON-1176745022 tempest-ServersNegativeTestJSON-1176745022-project-member] Lock "0484e878-6799-4030-9dcc-d62d32317744" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.089s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2026.631580] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "0484e878-6799-4030-9dcc-d62d32317744" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 204.259s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2026.631784] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 0484e878-6799-4030-9dcc-d62d32317744] During sync_power_state the instance has a pending task (deleting). Skip. [ 2026.631962] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "0484e878-6799-4030-9dcc-d62d32317744" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2046.350238] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "692af003-f18e-4cee-99fd-ef42d6e4b390" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2046.350585] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "692af003-f18e-4cee-99fd-ef42d6e4b390" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2046.853442] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 2047.375342] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2047.375659] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2047.377032] env[63593]: INFO nova.compute.claims [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2048.457441] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82b8938d-9adb-4296-b1de-13d954498fff {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2048.464760] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71e2e9f9-a1ec-4a2b-aa48-851ddffeefa7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2048.492823] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f6b1983-5894-424f-8106-641b5590f337 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2048.499171] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0490fd7-6bf2-4693-907c-dddb92fb4f27 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2048.512411] env[63593]: DEBUG nova.compute.provider_tree [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2049.015827] env[63593]: DEBUG nova.scheduler.client.report [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2049.520953] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.145s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2049.521500] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 2050.026243] env[63593]: DEBUG nova.compute.utils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2050.027658] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 2050.027829] env[63593]: DEBUG nova.network.neutron [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2050.066426] env[63593]: DEBUG nova.policy [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c0a03b9520a349cb99065ce54ec21927', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '87f6487499be40a9b8bf821e1bd4de80', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 2050.311645] env[63593]: DEBUG nova.network.neutron [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Successfully created port: 4f2f57c7-b639-4c88-a448-016422c578ed {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2050.534076] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 2051.542781] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 2051.567580] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2051.567819] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2051.567973] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2051.568163] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2051.568305] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2051.568449] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2051.568650] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2051.568802] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2051.568964] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2051.569137] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2051.569304] env[63593]: DEBUG nova.virt.hardware [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2051.570165] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed04ca56-09d7-476f-9263-379b927f6bf0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2051.577964] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7aa9e5b-3b89-42a3-ae3b-686c2471d5cf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2051.715805] env[63593]: DEBUG nova.compute.manager [req-c9e88bb3-8227-4149-94a6-503ea6f0818c req-b45ea801-acef-4f57-b399-175fda9b7c62 service nova] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Received event network-vif-plugged-4f2f57c7-b639-4c88-a448-016422c578ed {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 2051.716034] env[63593]: DEBUG oslo_concurrency.lockutils [req-c9e88bb3-8227-4149-94a6-503ea6f0818c req-b45ea801-acef-4f57-b399-175fda9b7c62 service nova] Acquiring lock "692af003-f18e-4cee-99fd-ef42d6e4b390-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2051.716238] env[63593]: DEBUG oslo_concurrency.lockutils [req-c9e88bb3-8227-4149-94a6-503ea6f0818c req-b45ea801-acef-4f57-b399-175fda9b7c62 service nova] Lock "692af003-f18e-4cee-99fd-ef42d6e4b390-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2051.716954] env[63593]: DEBUG oslo_concurrency.lockutils [req-c9e88bb3-8227-4149-94a6-503ea6f0818c req-b45ea801-acef-4f57-b399-175fda9b7c62 service nova] Lock "692af003-f18e-4cee-99fd-ef42d6e4b390-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2051.716954] env[63593]: DEBUG nova.compute.manager [req-c9e88bb3-8227-4149-94a6-503ea6f0818c req-b45ea801-acef-4f57-b399-175fda9b7c62 service nova] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] No waiting events found dispatching network-vif-plugged-4f2f57c7-b639-4c88-a448-016422c578ed {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2051.716954] env[63593]: WARNING nova.compute.manager [req-c9e88bb3-8227-4149-94a6-503ea6f0818c req-b45ea801-acef-4f57-b399-175fda9b7c62 service nova] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Received unexpected event network-vif-plugged-4f2f57c7-b639-4c88-a448-016422c578ed for instance with vm_state building and task_state spawning. [ 2051.797173] env[63593]: DEBUG nova.network.neutron [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Successfully updated port: 4f2f57c7-b639-4c88-a448-016422c578ed {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2052.299585] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "refresh_cache-692af003-f18e-4cee-99fd-ef42d6e4b390" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2052.299735] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "refresh_cache-692af003-f18e-4cee-99fd-ef42d6e4b390" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2052.299856] env[63593]: DEBUG nova.network.neutron [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2052.830426] env[63593]: DEBUG nova.network.neutron [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2052.951289] env[63593]: DEBUG nova.network.neutron [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Updating instance_info_cache with network_info: [{"id": "4f2f57c7-b639-4c88-a448-016422c578ed", "address": "fa:16:3e:68:42:9a", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4f2f57c7-b6", "ovs_interfaceid": "4f2f57c7-b639-4c88-a448-016422c578ed", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2053.453665] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "refresh_cache-692af003-f18e-4cee-99fd-ef42d6e4b390" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2053.453943] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Instance network_info: |[{"id": "4f2f57c7-b639-4c88-a448-016422c578ed", "address": "fa:16:3e:68:42:9a", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4f2f57c7-b6", "ovs_interfaceid": "4f2f57c7-b639-4c88-a448-016422c578ed", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 2053.454414] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:68:42:9a', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a15de394-0367-4921-a5c1-6ac8615e3283', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '4f2f57c7-b639-4c88-a448-016422c578ed', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2053.461798] env[63593]: DEBUG oslo.service.loopingcall [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2053.462018] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2053.462246] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-443a24f2-99dd-46c9-b59c-3c9af56fa28d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.481008] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2053.481008] env[63593]: value = "task-1368100" [ 2053.481008] env[63593]: _type = "Task" [ 2053.481008] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2053.488224] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368100, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2053.742408] env[63593]: DEBUG nova.compute.manager [req-f9d03c88-d4fb-4ef0-849c-45b3883752c0 req-a3a3f665-24c0-47e5-b989-9ec21fc86a0e service nova] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Received event network-changed-4f2f57c7-b639-4c88-a448-016422c578ed {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 2053.742715] env[63593]: DEBUG nova.compute.manager [req-f9d03c88-d4fb-4ef0-849c-45b3883752c0 req-a3a3f665-24c0-47e5-b989-9ec21fc86a0e service nova] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Refreshing instance network info cache due to event network-changed-4f2f57c7-b639-4c88-a448-016422c578ed. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 2053.743151] env[63593]: DEBUG oslo_concurrency.lockutils [req-f9d03c88-d4fb-4ef0-849c-45b3883752c0 req-a3a3f665-24c0-47e5-b989-9ec21fc86a0e service nova] Acquiring lock "refresh_cache-692af003-f18e-4cee-99fd-ef42d6e4b390" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2053.743344] env[63593]: DEBUG oslo_concurrency.lockutils [req-f9d03c88-d4fb-4ef0-849c-45b3883752c0 req-a3a3f665-24c0-47e5-b989-9ec21fc86a0e service nova] Acquired lock "refresh_cache-692af003-f18e-4cee-99fd-ef42d6e4b390" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2053.743580] env[63593]: DEBUG nova.network.neutron [req-f9d03c88-d4fb-4ef0-849c-45b3883752c0 req-a3a3f665-24c0-47e5-b989-9ec21fc86a0e service nova] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Refreshing network info cache for port 4f2f57c7-b639-4c88-a448-016422c578ed {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2053.991355] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368100, 'name': CreateVM_Task, 'duration_secs': 0.388005} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2053.991701] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2053.992258] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2053.992495] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2053.992878] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2053.993189] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b32da4c0-6fd5-4f4d-9a14-4a981728c12c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2053.997475] env[63593]: DEBUG oslo_vmware.api [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2053.997475] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52ac6917-d21d-8108-a004-e68a6b77d74e" [ 2053.997475] env[63593]: _type = "Task" [ 2053.997475] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2054.004619] env[63593]: DEBUG oslo_vmware.api [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52ac6917-d21d-8108-a004-e68a6b77d74e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2054.460258] env[63593]: DEBUG nova.network.neutron [req-f9d03c88-d4fb-4ef0-849c-45b3883752c0 req-a3a3f665-24c0-47e5-b989-9ec21fc86a0e service nova] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Updated VIF entry in instance network info cache for port 4f2f57c7-b639-4c88-a448-016422c578ed. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2054.460613] env[63593]: DEBUG nova.network.neutron [req-f9d03c88-d4fb-4ef0-849c-45b3883752c0 req-a3a3f665-24c0-47e5-b989-9ec21fc86a0e service nova] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Updating instance_info_cache with network_info: [{"id": "4f2f57c7-b639-4c88-a448-016422c578ed", "address": "fa:16:3e:68:42:9a", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap4f2f57c7-b6", "ovs_interfaceid": "4f2f57c7-b639-4c88-a448-016422c578ed", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2054.507477] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2054.507676] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2054.507890] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2054.963750] env[63593]: DEBUG oslo_concurrency.lockutils [req-f9d03c88-d4fb-4ef0-849c-45b3883752c0 req-a3a3f665-24c0-47e5-b989-9ec21fc86a0e service nova] Releasing lock "refresh_cache-692af003-f18e-4cee-99fd-ef42d6e4b390" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2057.019529] env[63593]: DEBUG oslo_concurrency.lockutils [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2066.742632] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2066.742995] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 2066.742995] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 2067.247206] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2067.247377] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2067.247491] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2067.247573] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2067.247695] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2067.247814] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2067.247958] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 2067.248187] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2067.248348] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2067.248495] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2067.248643] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2067.248767] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 2067.865158] env[63593]: WARNING oslo_vmware.rw_handles [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2067.865158] env[63593]: ERROR oslo_vmware.rw_handles [ 2067.865887] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/2e8f7777-c1a3-4520-a441-cec63753ae61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2067.867735] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2067.867977] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Copying Virtual Disk [datastore2] vmware_temp/2e8f7777-c1a3-4520-a441-cec63753ae61/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/2e8f7777-c1a3-4520-a441-cec63753ae61/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2067.868271] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-7c6f9f74-0c7b-4342-a788-13b699b14848 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2067.878409] env[63593]: DEBUG oslo_vmware.api [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 2067.878409] env[63593]: value = "task-1368101" [ 2067.878409] env[63593]: _type = "Task" [ 2067.878409] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2067.885869] env[63593]: DEBUG oslo_vmware.api [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': task-1368101, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2068.388642] env[63593]: DEBUG oslo_vmware.exceptions [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2068.388929] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2068.389496] env[63593]: ERROR nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2068.389496] env[63593]: Faults: ['InvalidArgument'] [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Traceback (most recent call last): [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] yield resources [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] self.driver.spawn(context, instance, image_meta, [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] self._fetch_image_if_missing(context, vi) [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] image_cache(vi, tmp_image_ds_loc) [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] vm_util.copy_virtual_disk( [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] session._wait_for_task(vmdk_copy_task) [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] return self.wait_for_task(task_ref) [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] return evt.wait() [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] result = hub.switch() [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] return self.greenlet.switch() [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] self.f(*self.args, **self.kw) [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] raise exceptions.translate_fault(task_info.error) [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Faults: ['InvalidArgument'] [ 2068.389496] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] [ 2068.390647] env[63593]: INFO nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Terminating instance [ 2068.391415] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2068.391613] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2068.391844] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-42913024-743b-4866-b7eb-252b0d4a43ea {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.393986] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2068.394186] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2068.394879] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0fd8aa72-bed8-4c55-9ccd-265d244758bb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.401514] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2068.401711] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-24fb3720-56aa-4267-ad7e-dd91b56619b8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.404762] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2068.404923] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2068.405584] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b8a2c170-b236-44c4-8f00-fb784690ecf6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.409890] env[63593]: DEBUG oslo_vmware.api [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 2068.409890] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5204bdb2-f2fa-8683-18ad-f32b25de87b5" [ 2068.409890] env[63593]: _type = "Task" [ 2068.409890] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2068.416870] env[63593]: DEBUG oslo_vmware.api [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]5204bdb2-f2fa-8683-18ad-f32b25de87b5, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2068.466574] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2068.466805] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2068.466984] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleting the datastore file [datastore2] 5f655ad5-1150-481c-9d34-f47eb8adcb9e {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2068.467263] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-662f95ca-3a77-42a9-91e4-8d42f06718f7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.472791] env[63593]: DEBUG oslo_vmware.api [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 2068.472791] env[63593]: value = "task-1368103" [ 2068.472791] env[63593]: _type = "Task" [ 2068.472791] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2068.480063] env[63593]: DEBUG oslo_vmware.api [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': task-1368103, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2068.920304] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2068.920679] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating directory with path [datastore2] vmware_temp/ebf2aae9-acd6-4a6b-a506-d5e25d4cede0/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2068.920799] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ed0b7308-ac3d-4748-bc3b-1a6244afd470 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.931705] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Created directory with path [datastore2] vmware_temp/ebf2aae9-acd6-4a6b-a506-d5e25d4cede0/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2068.931900] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Fetch image to [datastore2] vmware_temp/ebf2aae9-acd6-4a6b-a506-d5e25d4cede0/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2068.932143] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/ebf2aae9-acd6-4a6b-a506-d5e25d4cede0/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2068.932819] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ac764ea-c753-4b47-aeb5-d62a42b1d94a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.939175] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81c4c90f-da45-49ba-a467-73593e0ff684 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.947954] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acf5c441-5b7f-4f42-812b-5b1f32635334 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.981868] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fba312ee-e58e-422c-8b90-8152b01d9c09 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.988987] env[63593]: DEBUG oslo_vmware.api [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': task-1368103, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.076644} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2068.990460] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2068.990654] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2068.990823] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2068.990994] env[63593]: INFO nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2068.992780] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-d770ee7a-5e1e-42c4-9831-408b704020ff {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2068.994683] env[63593]: DEBUG nova.compute.claims [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 2068.994856] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2068.995076] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2069.020602] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2069.124937] env[63593]: DEBUG oslo_vmware.rw_handles [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ebf2aae9-acd6-4a6b-a506-d5e25d4cede0/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2069.188364] env[63593]: DEBUG oslo_vmware.rw_handles [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2069.188615] env[63593]: DEBUG oslo_vmware.rw_handles [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ebf2aae9-acd6-4a6b-a506-d5e25d4cede0/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2069.579815] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7e18821-639e-4782-81da-e39b8c51469e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.587047] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f367eb46-3ebb-4041-81c4-f1a315c99366 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.616160] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a45eaa6-1aa0-4e2c-a547-41c8a9e3d279 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.622705] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2cef7a05-03b3-44d3-8648-29f2acf23287 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2069.635201] env[63593]: DEBUG nova.compute.provider_tree [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2070.012480] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2070.137865] env[63593]: DEBUG nova.scheduler.client.report [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2070.643734] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.648s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2070.644338] env[63593]: ERROR nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2070.644338] env[63593]: Faults: ['InvalidArgument'] [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Traceback (most recent call last): [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] self.driver.spawn(context, instance, image_meta, [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] self._fetch_image_if_missing(context, vi) [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] image_cache(vi, tmp_image_ds_loc) [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] vm_util.copy_virtual_disk( [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] session._wait_for_task(vmdk_copy_task) [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] return self.wait_for_task(task_ref) [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] return evt.wait() [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] result = hub.switch() [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] return self.greenlet.switch() [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] self.f(*self.args, **self.kw) [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] raise exceptions.translate_fault(task_info.error) [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Faults: ['InvalidArgument'] [ 2070.644338] env[63593]: ERROR nova.compute.manager [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] [ 2070.645293] env[63593]: DEBUG nova.compute.utils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2070.646946] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Build of instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e was re-scheduled: A specified parameter was not correct: fileType [ 2070.646946] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 2070.647334] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 2070.647506] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 2070.647673] env[63593]: DEBUG nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2070.647848] env[63593]: DEBUG nova.network.neutron [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2071.012723] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2071.370402] env[63593]: DEBUG nova.network.neutron [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2071.873065] env[63593]: INFO nova.compute.manager [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Took 1.22 seconds to deallocate network for instance. [ 2072.920120] env[63593]: INFO nova.scheduler.client.report [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleted allocations for instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e [ 2073.428646] env[63593]: DEBUG oslo_concurrency.lockutils [None req-859a351c-ecfc-46a8-9798-19c8ef6479fe tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 604.713s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2073.428957] env[63593]: DEBUG oslo_concurrency.lockutils [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 408.077s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2073.429207] env[63593]: DEBUG oslo_concurrency.lockutils [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2073.429428] env[63593]: DEBUG oslo_concurrency.lockutils [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2073.429588] env[63593]: DEBUG oslo_concurrency.lockutils [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2073.432681] env[63593]: INFO nova.compute.manager [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Terminating instance [ 2073.434491] env[63593]: DEBUG nova.compute.manager [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2073.434667] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2073.434935] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f749b065-e69b-4c5a-b13b-2d59d770d4a5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2073.443526] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8228b8d3-0d88-4407-8c2b-7eaecd206b8b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2073.466496] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5f655ad5-1150-481c-9d34-f47eb8adcb9e could not be found. [ 2073.466698] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2073.466870] env[63593]: INFO nova.compute.manager [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2073.467149] env[63593]: DEBUG oslo.service.loopingcall [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2073.467377] env[63593]: DEBUG nova.compute.manager [-] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2073.467473] env[63593]: DEBUG nova.network.neutron [-] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2073.985527] env[63593]: DEBUG nova.network.neutron [-] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2074.489204] env[63593]: INFO nova.compute.manager [-] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] Took 1.02 seconds to deallocate network for instance. [ 2075.008527] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2075.517413] env[63593]: DEBUG oslo_concurrency.lockutils [None req-89865346-e527-4d28-8459-be4872a3d87e tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.088s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2075.518823] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 253.145s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2075.518823] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 5f655ad5-1150-481c-9d34-f47eb8adcb9e] During sync_power_state the instance has a pending task (deleting). Skip. [ 2075.518823] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "5f655ad5-1150-481c-9d34-f47eb8adcb9e" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2076.014066] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2076.014066] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11226}} [ 2076.517123] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] There are 0 instances to clean {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11235}} [ 2077.012628] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2077.515927] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2077.516347] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.001s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2077.516645] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2077.516906] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2077.518362] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae93b7f7-a84c-4ed9-876f-5d6bbc499f78 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.528678] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c2fad37-a17c-499f-8fb3-3d18409e74cf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.542531] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3142472b-15e6-4369-a6cd-3ab0c7ee58e5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.548399] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50f76634-fb0d-4162-88f3-0d86b532f624 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2077.575856] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181465MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2077.575987] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2077.576183] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2078.607284] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2078.607603] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 3e8b50e4-64fb-4566-b730-9acf5591cbda actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2078.607603] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 606babde-3951-4b7a-b4bd-f7c22ab09195 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2078.607733] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2078.607785] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 692af003-f18e-4cee-99fd-ef42d6e4b390 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2078.607996] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2078.608117] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1472MB phys_disk=200GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2078.625124] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing inventories for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 2078.637972] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Updating ProviderTree inventory for provider 7345f17f-3af2-4b0a-9521-0292dc691877 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 2078.638176] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Updating inventory in ProviderTree for provider 7345f17f-3af2-4b0a-9521-0292dc691877 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2078.648772] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing aggregate associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, aggregates: None {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 2078.665464] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Refreshing trait associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 2078.726906] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfd4cd88-aa8c-4bbd-8c76-aa505fce9dfb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2078.734350] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02eb2d56-ba30-40d0-9250-c35976460e1c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2078.763758] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a82fb36f-6120-47f8-8359-4e6a8a7e59a7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2078.770249] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16497c22-fe41-4b71-9250-523cd4aa1fe2 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2078.782759] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2079.286134] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2079.791432] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2079.791815] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.215s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2079.791879] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2079.791985] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances with incomplete migration {{(pid=63593) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11264}} [ 2080.294572] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2115.745583] env[63593]: WARNING oslo_vmware.rw_handles [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2115.745583] env[63593]: ERROR oslo_vmware.rw_handles [ 2115.746514] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/ebf2aae9-acd6-4a6b-a506-d5e25d4cede0/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2115.748758] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2115.749029] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Copying Virtual Disk [datastore2] vmware_temp/ebf2aae9-acd6-4a6b-a506-d5e25d4cede0/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/ebf2aae9-acd6-4a6b-a506-d5e25d4cede0/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2115.749312] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-1fce8eba-fea8-47c2-8559-c875285c2d62 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2115.757763] env[63593]: DEBUG oslo_vmware.api [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 2115.757763] env[63593]: value = "task-1368104" [ 2115.757763] env[63593]: _type = "Task" [ 2115.757763] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2115.765530] env[63593]: DEBUG oslo_vmware.api [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': task-1368104, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2116.269471] env[63593]: DEBUG oslo_vmware.exceptions [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2116.269772] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2116.270386] env[63593]: ERROR nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2116.270386] env[63593]: Faults: ['InvalidArgument'] [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Traceback (most recent call last): [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] yield resources [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] self.driver.spawn(context, instance, image_meta, [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] self._fetch_image_if_missing(context, vi) [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] image_cache(vi, tmp_image_ds_loc) [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] vm_util.copy_virtual_disk( [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] session._wait_for_task(vmdk_copy_task) [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] return self.wait_for_task(task_ref) [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] return evt.wait() [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] result = hub.switch() [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] return self.greenlet.switch() [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] self.f(*self.args, **self.kw) [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] raise exceptions.translate_fault(task_info.error) [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Faults: ['InvalidArgument'] [ 2116.270386] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] [ 2116.272417] env[63593]: INFO nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Terminating instance [ 2116.272484] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2116.273881] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2116.273881] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-c48fc3a8-8c00-4b3d-9895-beb291bbc752 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.275226] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2116.275410] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2116.276177] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43497306-db60-4c35-aa1b-aa96d2b226b5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.282788] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2116.282995] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-3a24c743-ad6d-4154-aa07-cb43aca1dbaa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.284990] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2116.285172] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2116.286114] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-941154ed-23b4-4f1d-9aca-3dd26f76f846 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.290511] env[63593]: DEBUG oslo_vmware.api [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2116.290511] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52462b8e-1178-39cd-7aef-8aed180be852" [ 2116.290511] env[63593]: _type = "Task" [ 2116.290511] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2116.297926] env[63593]: DEBUG oslo_vmware.api [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52462b8e-1178-39cd-7aef-8aed180be852, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2116.361260] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2116.361501] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2116.361649] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Deleting the datastore file [datastore2] 55ecc3e8-678b-4c92-820e-e3b3e01c7530 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2116.361932] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-9b971f95-7f49-4317-9a4f-84931142f030 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.368927] env[63593]: DEBUG oslo_vmware.api [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for the task: (returnval){ [ 2116.368927] env[63593]: value = "task-1368106" [ 2116.368927] env[63593]: _type = "Task" [ 2116.368927] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2116.376367] env[63593]: DEBUG oslo_vmware.api [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': task-1368106, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2116.800185] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2116.800565] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] vmware_temp/0c258e23-c1a6-458f-b23d-8ed4394663a5/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2116.800610] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-82c6821a-1642-4349-871a-b32d0ab26479 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.810904] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] vmware_temp/0c258e23-c1a6-458f-b23d-8ed4394663a5/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2116.811092] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Fetch image to [datastore2] vmware_temp/0c258e23-c1a6-458f-b23d-8ed4394663a5/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2116.811262] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/0c258e23-c1a6-458f-b23d-8ed4394663a5/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2116.811940] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c49fd7ed-2b19-4f08-b0f4-bb73ba9de573 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.817863] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92ac9e09-ed07-463a-a3c7-2b1724f34c68 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.826518] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1168a996-c65f-4cae-9709-947839f8f8d3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.855576] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40d15835-b7d6-4d1d-b271-177c11217065 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.860735] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-afd40c42-376d-464b-ba6f-eae94d10bff6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2116.876639] env[63593]: DEBUG oslo_vmware.api [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Task: {'id': task-1368106, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.081371} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2116.876868] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2116.877051] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2116.877224] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2116.877390] env[63593]: INFO nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2116.879422] env[63593]: DEBUG nova.compute.claims [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 2116.879587] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2116.879810] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2116.883883] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2116.949223] env[63593]: DEBUG oslo_vmware.rw_handles [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/0c258e23-c1a6-458f-b23d-8ed4394663a5/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2117.010876] env[63593]: DEBUG oslo_vmware.rw_handles [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2117.011115] env[63593]: DEBUG oslo_vmware.rw_handles [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/0c258e23-c1a6-458f-b23d-8ed4394663a5/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2117.454294] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3277d2b9-a589-468d-a3b1-99d7743aa574 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.461818] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ac72179-09d6-4b4b-9f62-dd0c24f6bf40 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.491198] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ca6d901-1391-4975-9bb9-d2f611dac220 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.498686] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a97a25d-aa41-4aab-9eae-ab633347faf6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2117.511753] env[63593]: DEBUG nova.compute.provider_tree [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2118.014655] env[63593]: DEBUG nova.scheduler.client.report [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2118.519600] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.640s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2118.520183] env[63593]: ERROR nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2118.520183] env[63593]: Faults: ['InvalidArgument'] [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Traceback (most recent call last): [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] self.driver.spawn(context, instance, image_meta, [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] self._fetch_image_if_missing(context, vi) [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] image_cache(vi, tmp_image_ds_loc) [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] vm_util.copy_virtual_disk( [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] session._wait_for_task(vmdk_copy_task) [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] return self.wait_for_task(task_ref) [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] return evt.wait() [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] result = hub.switch() [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] return self.greenlet.switch() [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] self.f(*self.args, **self.kw) [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] raise exceptions.translate_fault(task_info.error) [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Faults: ['InvalidArgument'] [ 2118.520183] env[63593]: ERROR nova.compute.manager [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] [ 2118.521290] env[63593]: DEBUG nova.compute.utils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2118.522723] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Build of instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 was re-scheduled: A specified parameter was not correct: fileType [ 2118.522723] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 2118.523099] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 2118.523276] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 2118.523469] env[63593]: DEBUG nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2118.523671] env[63593]: DEBUG nova.network.neutron [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2119.300350] env[63593]: DEBUG nova.network.neutron [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2119.803403] env[63593]: INFO nova.compute.manager [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Took 1.28 seconds to deallocate network for instance. [ 2120.837242] env[63593]: INFO nova.scheduler.client.report [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Deleted allocations for instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 [ 2121.346248] env[63593]: DEBUG oslo_concurrency.lockutils [None req-bfa4f0ad-813e-4006-84c9-21306f4b3fc2 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 510.355s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2121.346516] env[63593]: DEBUG oslo_concurrency.lockutils [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 314.557s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2121.346751] env[63593]: DEBUG oslo_concurrency.lockutils [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Acquiring lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2121.346963] env[63593]: DEBUG oslo_concurrency.lockutils [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2121.347147] env[63593]: DEBUG oslo_concurrency.lockutils [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2121.350361] env[63593]: INFO nova.compute.manager [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Terminating instance [ 2121.352471] env[63593]: DEBUG nova.compute.manager [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2121.352678] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2121.352946] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-de89572d-f2ad-4612-8ba2-fe7f42fd189a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2121.361750] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20278828-2adc-4cd9-820c-d00d5c191bbd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2121.384423] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 55ecc3e8-678b-4c92-820e-e3b3e01c7530 could not be found. [ 2121.384612] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2121.384800] env[63593]: INFO nova.compute.manager [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2121.385038] env[63593]: DEBUG oslo.service.loopingcall [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2121.385242] env[63593]: DEBUG nova.compute.manager [-] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2121.385335] env[63593]: DEBUG nova.network.neutron [-] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2121.902766] env[63593]: DEBUG nova.network.neutron [-] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2122.406311] env[63593]: INFO nova.compute.manager [-] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] Took 1.02 seconds to deallocate network for instance. [ 2123.431497] env[63593]: DEBUG oslo_concurrency.lockutils [None req-434f8b95-3023-465b-a8ac-62b74ec19ea4 tempest-AttachVolumeNegativeTest-50123938 tempest-AttachVolumeNegativeTest-50123938-project-member] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.085s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2123.432343] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 301.059s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2123.432565] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 55ecc3e8-678b-4c92-820e-e3b3e01c7530] During sync_power_state the instance has a pending task (deleting). Skip. [ 2123.432746] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "55ecc3e8-678b-4c92-820e-e3b3e01c7530" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2129.796767] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2129.797103] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 2129.797103] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 2130.301505] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2130.301712] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2130.301815] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2130.301926] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2130.302069] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 2130.302278] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2130.302435] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2130.302618] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2130.302781] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2130.302908] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 2131.013864] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2132.012974] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2136.009049] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2136.009049] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2137.012713] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2137.516109] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2137.516373] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2137.516545] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2137.516699] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2137.517617] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4784390-c517-4b26-adc3-718f3513ac98 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.525741] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-562d673e-6a3c-4787-89f0-2ee02c1a915f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.539675] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3656e65c-c1ee-4f7d-8be4-5940e228c1d5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.545853] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ec65aa8-8205-4928-9091-a6a1a3053710 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2137.574580] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181456MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2137.574744] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2137.574961] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2138.606727] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 3e8b50e4-64fb-4566-b730-9acf5591cbda actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2138.607028] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 606babde-3951-4b7a-b4bd-f7c22ab09195 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2138.607028] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2138.607135] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 692af003-f18e-4cee-99fd-ef42d6e4b390 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2138.607320] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 4 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2138.607455] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1280MB phys_disk=200GB used_disk=4GB total_vcpus=48 used_vcpus=4 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2138.663781] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db55f5d4-f5b7-4aef-9120-4319df9682f4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2138.670921] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d61e1c1-0b26-4898-94ce-83e766598a23 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2138.700812] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9710e5e0-4d9a-4244-bb95-eca77373d7d5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2138.707162] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e92655f-4ebf-4342-bd9e-9ebed71ef96d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2138.719569] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2139.224049] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2139.727828] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2139.728140] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.153s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2162.899143] env[63593]: WARNING oslo_vmware.rw_handles [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2162.899143] env[63593]: ERROR oslo_vmware.rw_handles [ 2162.899831] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/0c258e23-c1a6-458f-b23d-8ed4394663a5/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2162.901906] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2162.902192] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Copying Virtual Disk [datastore2] vmware_temp/0c258e23-c1a6-458f-b23d-8ed4394663a5/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/0c258e23-c1a6-458f-b23d-8ed4394663a5/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2162.902546] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-507b4373-cb09-48f8-8698-e28729d55aaa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2162.910545] env[63593]: DEBUG oslo_vmware.api [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2162.910545] env[63593]: value = "task-1368107" [ 2162.910545] env[63593]: _type = "Task" [ 2162.910545] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2162.918121] env[63593]: DEBUG oslo_vmware.api [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368107, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2163.421345] env[63593]: DEBUG oslo_vmware.exceptions [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2163.421641] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2163.422198] env[63593]: ERROR nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2163.422198] env[63593]: Faults: ['InvalidArgument'] [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Traceback (most recent call last): [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] yield resources [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] self.driver.spawn(context, instance, image_meta, [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] self._fetch_image_if_missing(context, vi) [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] image_cache(vi, tmp_image_ds_loc) [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] vm_util.copy_virtual_disk( [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] session._wait_for_task(vmdk_copy_task) [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] return self.wait_for_task(task_ref) [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] return evt.wait() [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] result = hub.switch() [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] return self.greenlet.switch() [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] self.f(*self.args, **self.kw) [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] raise exceptions.translate_fault(task_info.error) [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Faults: ['InvalidArgument'] [ 2163.422198] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] [ 2163.423371] env[63593]: INFO nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Terminating instance [ 2163.424066] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2163.424274] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2163.424509] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8ff3f1d8-a2a5-4229-a229-18212d1e38b0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.426629] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2163.426820] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2163.427540] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c856bb08-2956-43e7-9cdb-856a9a37d6d8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.434549] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2163.435526] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-b057d938-d5b4-4332-8d33-f60f044b171a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.436994] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2163.437067] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2163.437663] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-61a3ad9c-263f-4545-b9d7-3c6f11303ed7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.442295] env[63593]: DEBUG oslo_vmware.api [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Waiting for the task: (returnval){ [ 2163.442295] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52373f60-8463-487f-fb83-5b4e467428ca" [ 2163.442295] env[63593]: _type = "Task" [ 2163.442295] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2163.450039] env[63593]: DEBUG oslo_vmware.api [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52373f60-8463-487f-fb83-5b4e467428ca, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2163.503644] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2163.503863] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2163.504079] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleting the datastore file [datastore2] 3e8b50e4-64fb-4566-b730-9acf5591cbda {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2163.504341] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-c4d5d5eb-0672-4e28-b37b-985b467e0039 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.510893] env[63593]: DEBUG oslo_vmware.api [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2163.510893] env[63593]: value = "task-1368109" [ 2163.510893] env[63593]: _type = "Task" [ 2163.510893] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2163.518446] env[63593]: DEBUG oslo_vmware.api [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368109, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2163.952564] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2163.952945] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Creating directory with path [datastore2] vmware_temp/61713dc0-e97d-448c-9a78-260097d041e3/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2163.953124] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b9806b2c-3910-4c3e-a876-2db7cc23dbd4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.963912] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Created directory with path [datastore2] vmware_temp/61713dc0-e97d-448c-9a78-260097d041e3/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2163.964117] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Fetch image to [datastore2] vmware_temp/61713dc0-e97d-448c-9a78-260097d041e3/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2163.964290] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/61713dc0-e97d-448c-9a78-260097d041e3/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2163.965008] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9b7f15c-7737-4a97-a97e-4e9f749de674 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.971424] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4e73a8c-7c50-4074-b79f-888fe2634c34 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2163.980084] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34e8c5c7-9c8e-4e57-8a2c-e70738c5a9ea {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.009191] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb066916-7c7f-4cf0-9c9e-a54f916d2b9b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.019316] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-1dcbb3b9-1b10-4b2b-bc05-0ed0ab5df294 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.020854] env[63593]: DEBUG oslo_vmware.api [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368109, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.063269} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2164.021101] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2164.021275] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2164.021441] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2164.021609] env[63593]: INFO nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Took 0.59 seconds to destroy the instance on the hypervisor. [ 2164.023663] env[63593]: DEBUG nova.compute.claims [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 2164.023850] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2164.024100] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2164.044560] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2164.096841] env[63593]: DEBUG oslo_vmware.rw_handles [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/61713dc0-e97d-448c-9a78-260097d041e3/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2164.161722] env[63593]: DEBUG oslo_vmware.rw_handles [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2164.161911] env[63593]: DEBUG oslo_vmware.rw_handles [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/61713dc0-e97d-448c-9a78-260097d041e3/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2164.581516] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8918d021-3c51-4faa-90b6-bc4308cdc37f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.589169] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c53b873-4934-4b5a-868b-1bf8071c1cb4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.619282] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93013a60-c28b-46fd-a238-70298dee51b9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.626498] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37b95074-f488-4979-b014-0c4b75a5459e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2164.638946] env[63593]: DEBUG nova.compute.provider_tree [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2165.141643] env[63593]: DEBUG nova.scheduler.client.report [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2165.647080] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.622s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2165.647294] env[63593]: ERROR nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2165.647294] env[63593]: Faults: ['InvalidArgument'] [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Traceback (most recent call last): [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] self.driver.spawn(context, instance, image_meta, [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] self._fetch_image_if_missing(context, vi) [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] image_cache(vi, tmp_image_ds_loc) [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] vm_util.copy_virtual_disk( [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] session._wait_for_task(vmdk_copy_task) [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] return self.wait_for_task(task_ref) [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] return evt.wait() [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] result = hub.switch() [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] return self.greenlet.switch() [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] self.f(*self.args, **self.kw) [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] raise exceptions.translate_fault(task_info.error) [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Faults: ['InvalidArgument'] [ 2165.647294] env[63593]: ERROR nova.compute.manager [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] [ 2165.648572] env[63593]: DEBUG nova.compute.utils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2165.649783] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Build of instance 3e8b50e4-64fb-4566-b730-9acf5591cbda was re-scheduled: A specified parameter was not correct: fileType [ 2165.649783] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 2165.650185] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 2165.650362] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 2165.650527] env[63593]: DEBUG nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2165.650715] env[63593]: DEBUG nova.network.neutron [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2166.397654] env[63593]: DEBUG nova.network.neutron [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2166.901151] env[63593]: INFO nova.compute.manager [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Took 1.25 seconds to deallocate network for instance. [ 2167.937434] env[63593]: INFO nova.scheduler.client.report [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted allocations for instance 3e8b50e4-64fb-4566-b730-9acf5591cbda [ 2168.450061] env[63593]: DEBUG oslo_concurrency.lockutils [None req-e72fa89b-0621-451d-a822-4c4c6f7438ec tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 514.627s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2168.450061] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 346.076s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2168.450061] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] During sync_power_state the instance has a pending task (spawning). Skip. [ 2168.450061] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2168.450318] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 318.190s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2168.450467] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "3e8b50e4-64fb-4566-b730-9acf5591cbda-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2168.450674] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2168.450855] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2168.453790] env[63593]: INFO nova.compute.manager [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Terminating instance [ 2168.455978] env[63593]: DEBUG nova.compute.manager [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2168.456190] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2168.456445] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4f770f50-efbe-473e-9cac-919c2707019e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2168.464617] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1608d1b-7ddc-4213-8445-ef19d196fafe {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2168.487516] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3e8b50e4-64fb-4566-b730-9acf5591cbda could not be found. [ 2168.487698] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2168.487868] env[63593]: INFO nova.compute.manager [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2168.488112] env[63593]: DEBUG oslo.service.loopingcall [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2168.488312] env[63593]: DEBUG nova.compute.manager [-] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2168.488404] env[63593]: DEBUG nova.network.neutron [-] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2169.006370] env[63593]: DEBUG nova.network.neutron [-] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2169.509591] env[63593]: INFO nova.compute.manager [-] [instance: 3e8b50e4-64fb-4566-b730-9acf5591cbda] Took 1.02 seconds to deallocate network for instance. [ 2170.536295] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0f82f8c6-7cb8-40ef-83ba-ad97313574be tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "3e8b50e4-64fb-4566-b730-9acf5591cbda" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.086s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2188.731128] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2188.731128] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 2188.731128] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 2189.235272] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2189.235444] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2189.235577] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2189.235705] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 2189.235914] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2189.236085] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2189.236237] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2189.236387] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2189.236511] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 2193.013305] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2193.013680] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2198.009308] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2198.011921] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2198.516176] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2198.516425] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2198.516597] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2198.516747] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2198.517677] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-899a3a8c-c8cf-4f8a-85d9-e474e3d36d62 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2198.526863] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72bd5442-ce24-439c-ae0b-65442f97c26c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2198.540112] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be8c1067-6e65-4e62-8a2f-30713563e310 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2198.545929] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c18b1e8b-5c7a-4156-ad0a-4e91c42209c7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2198.573259] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181468MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2198.573409] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2198.573597] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2199.599543] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 606babde-3951-4b7a-b4bd-f7c22ab09195 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2199.599832] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2199.599832] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 692af003-f18e-4cee-99fd-ef42d6e4b390 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2199.599994] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 3 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2199.600148] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1088MB phys_disk=200GB used_disk=3GB total_vcpus=48 used_vcpus=3 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2199.645964] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9752f781-1438-40c2-8380-bba8af37ab2f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2199.653307] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9532a17d-973e-4bbf-9846-0a767058808c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2199.682847] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bdb62e6-9eda-4c86-8c01-9bb946026136 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2199.689815] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a5e4b9b-ca18-45a6-898e-47849899640a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2199.702371] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2200.204980] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2200.709926] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2200.710333] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.137s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2212.915071] env[63593]: WARNING oslo_vmware.rw_handles [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2212.915071] env[63593]: ERROR oslo_vmware.rw_handles [ 2212.915841] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/61713dc0-e97d-448c-9a78-260097d041e3/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2212.917636] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2212.917872] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Copying Virtual Disk [datastore2] vmware_temp/61713dc0-e97d-448c-9a78-260097d041e3/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/61713dc0-e97d-448c-9a78-260097d041e3/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2212.918172] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-13395699-b8aa-4eb7-bd64-6ae871f1c1d9 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2212.925952] env[63593]: DEBUG oslo_vmware.api [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Waiting for the task: (returnval){ [ 2212.925952] env[63593]: value = "task-1368110" [ 2212.925952] env[63593]: _type = "Task" [ 2212.925952] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2212.933353] env[63593]: DEBUG oslo_vmware.api [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Task: {'id': task-1368110, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2213.435974] env[63593]: DEBUG oslo_vmware.exceptions [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2213.436423] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2213.436991] env[63593]: ERROR nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2213.436991] env[63593]: Faults: ['InvalidArgument'] [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Traceback (most recent call last): [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] yield resources [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] self.driver.spawn(context, instance, image_meta, [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] self._fetch_image_if_missing(context, vi) [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] image_cache(vi, tmp_image_ds_loc) [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] vm_util.copy_virtual_disk( [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] session._wait_for_task(vmdk_copy_task) [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] return self.wait_for_task(task_ref) [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] return evt.wait() [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] result = hub.switch() [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] return self.greenlet.switch() [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] self.f(*self.args, **self.kw) [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] raise exceptions.translate_fault(task_info.error) [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Faults: ['InvalidArgument'] [ 2213.436991] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] [ 2213.438389] env[63593]: INFO nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Terminating instance [ 2213.440404] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2213.440671] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2213.440963] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2213.441169] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2213.441854] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d39514a-0699-4b4e-863c-33fb1e5dde17 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.444297] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-2188764f-6faa-4371-b89c-5f0e5c92bc8e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.449964] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2213.450209] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0ab02c87-af2c-4347-9b31-8a422bab84b1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.452259] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2213.452455] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2213.453433] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-09ceb0db-d547-4ff3-b1d4-aaeb0e1ba84f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.458071] env[63593]: DEBUG oslo_vmware.api [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 2213.458071] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52456dee-4b46-3ae6-0df9-2a02db163d7e" [ 2213.458071] env[63593]: _type = "Task" [ 2213.458071] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2213.472077] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2213.472290] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating directory with path [datastore2] vmware_temp/b6dbc29d-ee65-4ecd-8208-ef23a4a7fb37/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2213.472498] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-fc729abb-635c-4888-88de-1316016a4796 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.490606] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Created directory with path [datastore2] vmware_temp/b6dbc29d-ee65-4ecd-8208-ef23a4a7fb37/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2213.490794] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Fetch image to [datastore2] vmware_temp/b6dbc29d-ee65-4ecd-8208-ef23a4a7fb37/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2213.490956] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/b6dbc29d-ee65-4ecd-8208-ef23a4a7fb37/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2213.491677] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e22dd2e-2e84-4293-a06f-f5f166f43fe1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.498036] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1368529f-2f97-41ee-a0d1-bcb6237f2920 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.507621] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-995cac90-9248-4d32-a6fb-4549671b6390 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.514664] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2213.514853] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2213.515033] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Deleting the datastore file [datastore2] 606babde-3951-4b7a-b4bd-f7c22ab09195 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2213.515610] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-abdb818c-1976-46e0-9729-6650cff39e6e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.543956] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4db0ef81-65f3-42af-9cd2-a25f06b56c31 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.546296] env[63593]: DEBUG oslo_vmware.api [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Waiting for the task: (returnval){ [ 2213.546296] env[63593]: value = "task-1368112" [ 2213.546296] env[63593]: _type = "Task" [ 2213.546296] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2213.550811] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-ed9cb540-c65f-4388-8440-f7c7832f5edf {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2213.554801] env[63593]: DEBUG oslo_vmware.api [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Task: {'id': task-1368112, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2213.575074] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2213.623507] env[63593]: DEBUG oslo_vmware.rw_handles [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/b6dbc29d-ee65-4ecd-8208-ef23a4a7fb37/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2213.684963] env[63593]: DEBUG oslo_vmware.rw_handles [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2213.685202] env[63593]: DEBUG oslo_vmware.rw_handles [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/b6dbc29d-ee65-4ecd-8208-ef23a4a7fb37/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2214.055759] env[63593]: DEBUG oslo_vmware.api [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Task: {'id': task-1368112, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.067397} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2214.056178] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2214.056178] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2214.056345] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2214.056472] env[63593]: INFO nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Took 0.62 seconds to destroy the instance on the hypervisor. [ 2214.058692] env[63593]: DEBUG nova.compute.claims [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 2214.058861] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2214.059087] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2214.613856] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09e6eac2-4248-4043-b798-3d140e3c4970 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2214.620970] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77f66b3f-6c4b-4b47-84c3-686769fb1028 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2214.650736] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c137b83f-d3a1-4e22-9b6d-54355bb717c3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2214.657316] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8defb211-1beb-4b2f-82f2-07da3d9b6720 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2214.669552] env[63593]: DEBUG nova.compute.provider_tree [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2215.172918] env[63593]: DEBUG nova.scheduler.client.report [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2215.678410] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.619s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2215.678999] env[63593]: ERROR nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2215.678999] env[63593]: Faults: ['InvalidArgument'] [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Traceback (most recent call last): [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] self.driver.spawn(context, instance, image_meta, [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] self._fetch_image_if_missing(context, vi) [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] image_cache(vi, tmp_image_ds_loc) [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] vm_util.copy_virtual_disk( [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] session._wait_for_task(vmdk_copy_task) [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] return self.wait_for_task(task_ref) [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] return evt.wait() [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] result = hub.switch() [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] return self.greenlet.switch() [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] self.f(*self.args, **self.kw) [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] raise exceptions.translate_fault(task_info.error) [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Faults: ['InvalidArgument'] [ 2215.678999] env[63593]: ERROR nova.compute.manager [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] [ 2215.680016] env[63593]: DEBUG nova.compute.utils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2215.681527] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Build of instance 606babde-3951-4b7a-b4bd-f7c22ab09195 was re-scheduled: A specified parameter was not correct: fileType [ 2215.681527] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 2215.681889] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 2215.682083] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 2215.682257] env[63593]: DEBUG nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2215.682418] env[63593]: DEBUG nova.network.neutron [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2216.439577] env[63593]: DEBUG nova.network.neutron [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2216.943336] env[63593]: INFO nova.compute.manager [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Took 1.26 seconds to deallocate network for instance. [ 2217.981815] env[63593]: INFO nova.scheduler.client.report [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Deleted allocations for instance 606babde-3951-4b7a-b4bd-f7c22ab09195 [ 2218.490314] env[63593]: DEBUG oslo_concurrency.lockutils [None req-90044857-e5cf-4f30-96f6-6f9d9df33a4d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 402.033s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2218.490576] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 396.117s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2218.490774] env[63593]: INFO nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] During sync_power_state the instance has a pending task (spawning). Skip. [ 2218.490953] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2218.491207] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 206.245s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2218.491443] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Acquiring lock "606babde-3951-4b7a-b4bd-f7c22ab09195-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2218.491635] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2218.491792] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2218.495366] env[63593]: INFO nova.compute.manager [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Terminating instance [ 2218.497525] env[63593]: DEBUG nova.compute.manager [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2218.497717] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2218.497988] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d4592c82-55f2-44d9-a64a-3cce0e48355f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2218.506761] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-500c0ee8-daf5-4e2a-89e7-2bb25cd8eb73 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2218.529052] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 606babde-3951-4b7a-b4bd-f7c22ab09195 could not be found. [ 2218.529265] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2218.529443] env[63593]: INFO nova.compute.manager [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2218.529681] env[63593]: DEBUG oslo.service.loopingcall [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2218.529892] env[63593]: DEBUG nova.compute.manager [-] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2218.529988] env[63593]: DEBUG nova.network.neutron [-] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2219.050654] env[63593]: DEBUG nova.network.neutron [-] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2219.553604] env[63593]: INFO nova.compute.manager [-] [instance: 606babde-3951-4b7a-b4bd-f7c22ab09195] Took 1.02 seconds to deallocate network for instance. [ 2220.580191] env[63593]: DEBUG oslo_concurrency.lockutils [None req-a2c1545e-467a-49af-a1c6-5da0f638e92d tempest-InstanceActionsV221TestJSON-1450796412 tempest-InstanceActionsV221TestJSON-1450796412-project-member] Lock "606babde-3951-4b7a-b4bd-f7c22ab09195" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.089s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2241.806216] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "692af003-f18e-4cee-99fd-ef42d6e4b390" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2248.713667] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2248.714098] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2248.714151] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 2249.013274] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2249.013479] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 2249.013559] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 2249.517406] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2249.517573] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2249.517736] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 2249.517983] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2251.013859] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2253.014552] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2254.014735] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2259.008874] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2259.009203] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2259.334118] env[63593]: WARNING oslo_vmware.rw_handles [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2259.334118] env[63593]: ERROR oslo_vmware.rw_handles [ 2259.334677] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/b6dbc29d-ee65-4ecd-8208-ef23a4a7fb37/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2259.336712] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2259.336997] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Copying Virtual Disk [datastore2] vmware_temp/b6dbc29d-ee65-4ecd-8208-ef23a4a7fb37/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/b6dbc29d-ee65-4ecd-8208-ef23a4a7fb37/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2259.337334] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-ed6f7682-e812-426d-9536-7af7125c60e8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2259.345565] env[63593]: DEBUG oslo_vmware.api [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 2259.345565] env[63593]: value = "task-1368113" [ 2259.345565] env[63593]: _type = "Task" [ 2259.345565] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2259.353341] env[63593]: DEBUG oslo_vmware.api [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': task-1368113, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2259.855611] env[63593]: DEBUG oslo_vmware.exceptions [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2259.855914] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2259.856549] env[63593]: ERROR nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2259.856549] env[63593]: Faults: ['InvalidArgument'] [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Traceback (most recent call last): [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] yield resources [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] self.driver.spawn(context, instance, image_meta, [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] self._fetch_image_if_missing(context, vi) [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] image_cache(vi, tmp_image_ds_loc) [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] vm_util.copy_virtual_disk( [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] session._wait_for_task(vmdk_copy_task) [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] return self.wait_for_task(task_ref) [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] return evt.wait() [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] result = hub.switch() [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] return self.greenlet.switch() [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] self.f(*self.args, **self.kw) [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] raise exceptions.translate_fault(task_info.error) [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Faults: ['InvalidArgument'] [ 2259.856549] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] [ 2259.857791] env[63593]: INFO nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Terminating instance [ 2259.858603] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2259.858805] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2259.859047] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a00d06f4-fd52-4c12-be8a-4f00551cc6f8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2259.861062] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2259.861256] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2259.862070] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33f8f61a-a24a-4505-8ff2-9ba5df4ebc03 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2259.868309] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2259.868497] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-22771830-6bed-4b1b-ba24-6d601a050803 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2259.870446] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2259.870610] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2259.871503] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a4710f04-729d-4f8b-9078-187d2265ec10 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2259.875790] env[63593]: DEBUG oslo_vmware.api [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2259.875790] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52a4d48e-703d-6814-42de-ec8be931f5f1" [ 2259.875790] env[63593]: _type = "Task" [ 2259.875790] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2259.882524] env[63593]: DEBUG oslo_vmware.api [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52a4d48e-703d-6814-42de-ec8be931f5f1, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2259.932582] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2259.932818] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2259.932957] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleting the datastore file [datastore2] 053c6990-4a0f-4a9c-abfc-1b4b984f42cd {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2259.933218] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-b0adfa37-e143-44fd-843a-157c65b6bf1c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2259.938783] env[63593]: DEBUG oslo_vmware.api [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for the task: (returnval){ [ 2259.938783] env[63593]: value = "task-1368115" [ 2259.938783] env[63593]: _type = "Task" [ 2259.938783] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2259.946131] env[63593]: DEBUG oslo_vmware.api [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': task-1368115, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2260.012814] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2260.387058] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2260.387058] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] vmware_temp/7fdd61eb-c818-4160-b5cb-69f7f3c5f800/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2260.387058] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0a014a6d-39b7-4a32-ae8a-8323bb4dcd8d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.397686] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] vmware_temp/7fdd61eb-c818-4160-b5cb-69f7f3c5f800/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2260.397866] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Fetch image to [datastore2] vmware_temp/7fdd61eb-c818-4160-b5cb-69f7f3c5f800/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2260.398045] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/7fdd61eb-c818-4160-b5cb-69f7f3c5f800/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2260.398728] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20092d73-0996-4f17-a74c-084f09130044 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.404967] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fee81b42-dc6f-4d39-8827-76b70538c4ad {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.413419] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ecb71e2-a4d8-41a5-b40c-1e930c209966 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.446025] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a169440-b4de-4539-86b3-a75e6cd7a705 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.452432] env[63593]: DEBUG oslo_vmware.api [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Task: {'id': task-1368115, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.077382} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2260.453766] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2260.453981] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2260.454176] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2260.454346] env[63593]: INFO nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Took 0.59 seconds to destroy the instance on the hypervisor. [ 2260.456057] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-7e9d7fc7-0353-4134-aa4b-d61164d8928b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2260.457809] env[63593]: DEBUG nova.compute.claims [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 2260.457984] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2260.458213] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2260.481020] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2260.515540] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2260.529901] env[63593]: DEBUG oslo_vmware.rw_handles [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7fdd61eb-c818-4160-b5cb-69f7f3c5f800/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2260.590704] env[63593]: DEBUG oslo_vmware.rw_handles [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2260.590897] env[63593]: DEBUG oslo_vmware.rw_handles [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7fdd61eb-c818-4160-b5cb-69f7f3c5f800/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2261.007316] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3a94659-6ede-404f-b286-ed04979724aa {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.015247] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-edba7a83-25b4-4be0-b79c-f762fc8f265b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.043971] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49236521-72e9-4959-b865-9289c36f69a3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.050736] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-962b9bac-c064-437e-bb56-a4b3eefcaa78 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2261.063256] env[63593]: DEBUG nova.compute.provider_tree [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2261.565976] env[63593]: DEBUG nova.scheduler.client.report [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2262.071983] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.614s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2262.072616] env[63593]: ERROR nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2262.072616] env[63593]: Faults: ['InvalidArgument'] [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Traceback (most recent call last): [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] self.driver.spawn(context, instance, image_meta, [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] self._fetch_image_if_missing(context, vi) [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] image_cache(vi, tmp_image_ds_loc) [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] vm_util.copy_virtual_disk( [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] session._wait_for_task(vmdk_copy_task) [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] return self.wait_for_task(task_ref) [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] return evt.wait() [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] result = hub.switch() [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] return self.greenlet.switch() [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] self.f(*self.args, **self.kw) [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] raise exceptions.translate_fault(task_info.error) [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Faults: ['InvalidArgument'] [ 2262.072616] env[63593]: ERROR nova.compute.manager [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] [ 2262.073937] env[63593]: DEBUG nova.compute.utils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2262.074973] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 1.559s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2262.074973] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2262.075137] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2262.075573] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Build of instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd was re-scheduled: A specified parameter was not correct: fileType [ 2262.075573] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 2262.075932] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 2262.076118] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 2262.076318] env[63593]: DEBUG nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2262.076503] env[63593]: DEBUG nova.network.neutron [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2262.078642] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf8df09a-7315-49df-97ba-d6b1d54b4bfd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.087196] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b5b973e-fbf6-42d2-828c-986f49cf3f15 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.100908] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdc3bb18-6317-4916-89f7-4c361cae1a62 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.106915] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-017c624c-130d-4a4c-b68a-0dfc434a49fd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2262.135375] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181471MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2262.135532] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2262.135728] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2262.864176] env[63593]: DEBUG nova.network.neutron [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2263.366639] env[63593]: INFO nova.compute.manager [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Took 1.29 seconds to deallocate network for instance. [ 2263.659246] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 2263.659477] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 692af003-f18e-4cee-99fd-ef42d6e4b390 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2263.659597] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 1 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2263.659743] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=704MB phys_disk=200GB used_disk=1GB total_vcpus=48 used_vcpus=1 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2263.695580] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4551ad2-a55b-474e-a246-d7be6f64dff8 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2263.703037] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47beb699-58eb-4f0c-9779-613e1585317f {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2263.731231] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a10ee85e-1823-4306-902d-2b3c0b872f54 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2263.737745] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4357a7f-84c3-4d65-9667-94d6647a7e83 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2263.750174] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2264.252873] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2264.395189] env[63593]: INFO nova.scheduler.client.report [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Deleted allocations for instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd [ 2264.757149] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2264.757371] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.622s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2264.903324] env[63593]: DEBUG oslo_concurrency.lockutils [None req-d6da4e4c-ae62-4992-96ec-e791de138eb8 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 403.703s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2264.903579] env[63593]: DEBUG oslo_concurrency.lockutils [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 207.884s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2264.903804] env[63593]: DEBUG oslo_concurrency.lockutils [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Acquiring lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2264.904082] env[63593]: DEBUG oslo_concurrency.lockutils [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2264.904257] env[63593]: DEBUG oslo_concurrency.lockutils [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2264.907091] env[63593]: INFO nova.compute.manager [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Terminating instance [ 2264.908836] env[63593]: DEBUG nova.compute.manager [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2264.909030] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2264.909282] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b141905d-280d-4d10-ae33-6d2b64886c0c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.917782] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5c47aab-125d-4a2d-829f-8241aef5f86e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2264.939932] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 053c6990-4a0f-4a9c-abfc-1b4b984f42cd could not be found. [ 2264.940132] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2264.940306] env[63593]: INFO nova.compute.manager [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2264.940535] env[63593]: DEBUG oslo.service.loopingcall [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2264.940723] env[63593]: DEBUG nova.compute.manager [-] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2264.940815] env[63593]: DEBUG nova.network.neutron [-] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2265.458967] env[63593]: DEBUG nova.network.neutron [-] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2265.961537] env[63593]: INFO nova.compute.manager [-] [instance: 053c6990-4a0f-4a9c-abfc-1b4b984f42cd] Took 1.02 seconds to deallocate network for instance. [ 2266.985798] env[63593]: DEBUG oslo_concurrency.lockutils [None req-01947211-4ce3-4279-89b7-528327287ce7 tempest-ServersTestJSON-1122993187 tempest-ServersTestJSON-1122993187-project-member] Lock "053c6990-4a0f-4a9c-abfc-1b4b984f42cd" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.082s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2307.962479] env[63593]: WARNING oslo_vmware.rw_handles [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2307.962479] env[63593]: ERROR oslo_vmware.rw_handles [ 2307.964136] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/7fdd61eb-c818-4160-b5cb-69f7f3c5f800/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2307.964920] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2307.965183] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Copying Virtual Disk [datastore2] vmware_temp/7fdd61eb-c818-4160-b5cb-69f7f3c5f800/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/7fdd61eb-c818-4160-b5cb-69f7f3c5f800/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2307.965516] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-a8bc73fa-2716-4740-ba1f-95546c781766 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2307.975267] env[63593]: DEBUG oslo_vmware.api [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2307.975267] env[63593]: value = "task-1368116" [ 2307.975267] env[63593]: _type = "Task" [ 2307.975267] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2307.983017] env[63593]: DEBUG oslo_vmware.api [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368116, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2308.485906] env[63593]: DEBUG oslo_vmware.exceptions [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2308.486231] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2308.486764] env[63593]: ERROR nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2308.486764] env[63593]: Faults: ['InvalidArgument'] [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Traceback (most recent call last): [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] yield resources [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] self.driver.spawn(context, instance, image_meta, [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] self._fetch_image_if_missing(context, vi) [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] image_cache(vi, tmp_image_ds_loc) [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] vm_util.copy_virtual_disk( [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] session._wait_for_task(vmdk_copy_task) [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] return self.wait_for_task(task_ref) [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] return evt.wait() [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] result = hub.switch() [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] return self.greenlet.switch() [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] self.f(*self.args, **self.kw) [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] raise exceptions.translate_fault(task_info.error) [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Faults: ['InvalidArgument'] [ 2308.486764] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] [ 2308.487986] env[63593]: INFO nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Terminating instance [ 2308.490021] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2308.490160] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2308.490884] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6bf4eee9-e694-4b4d-af3e-03df042afbd4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.497541] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2308.497751] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-4a12d403-1b1b-4efb-abd0-a28e74a0752d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.564056] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2308.564307] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2308.564506] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleting the datastore file [datastore2] 692af003-f18e-4cee-99fd-ef42d6e4b390 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2308.564870] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-20eaed24-b22b-4c5b-9dd0-74870ea49eac {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.570958] env[63593]: DEBUG oslo_vmware.api [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2308.570958] env[63593]: value = "task-1368118" [ 2308.570958] env[63593]: _type = "Task" [ 2308.570958] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2308.578044] env[63593]: DEBUG oslo_vmware.api [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368118, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2309.080932] env[63593]: DEBUG oslo_vmware.api [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368118, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.068566} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2309.081380] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2309.081380] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2309.081530] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2309.081695] env[63593]: INFO nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Took 0.59 seconds to destroy the instance on the hypervisor. [ 2309.083792] env[63593]: DEBUG nova.compute.claims [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 2309.083960] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2309.084187] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2309.621778] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d588be02-f91b-4afe-951d-19719d81a876 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2309.629660] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f0ea76e-d33a-47dc-8f63-c090ae5c1f71 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2309.658518] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fe4f714-5cb8-4650-a513-971e161549e3 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2309.665153] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7712fc99-4aa6-4e4c-8418-7bbf3d1f1b42 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2309.677537] env[63593]: DEBUG nova.compute.provider_tree [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2310.180100] env[63593]: DEBUG nova.scheduler.client.report [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2310.685752] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.601s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2310.686331] env[63593]: ERROR nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2310.686331] env[63593]: Faults: ['InvalidArgument'] [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Traceback (most recent call last): [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] self.driver.spawn(context, instance, image_meta, [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] self._fetch_image_if_missing(context, vi) [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] image_cache(vi, tmp_image_ds_loc) [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] vm_util.copy_virtual_disk( [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] session._wait_for_task(vmdk_copy_task) [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] return self.wait_for_task(task_ref) [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] return evt.wait() [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] result = hub.switch() [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] return self.greenlet.switch() [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] self.f(*self.args, **self.kw) [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] raise exceptions.translate_fault(task_info.error) [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Faults: ['InvalidArgument'] [ 2310.686331] env[63593]: ERROR nova.compute.manager [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] [ 2310.687513] env[63593]: DEBUG nova.compute.utils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2310.688912] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Build of instance 692af003-f18e-4cee-99fd-ef42d6e4b390 was re-scheduled: A specified parameter was not correct: fileType [ 2310.688912] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 2310.689296] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 2310.689469] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 2310.689657] env[63593]: DEBUG nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2310.689827] env[63593]: DEBUG nova.network.neutron [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2311.423900] env[63593]: DEBUG nova.network.neutron [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2311.926790] env[63593]: INFO nova.compute.manager [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Took 1.24 seconds to deallocate network for instance. [ 2312.757400] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2312.757785] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 2312.757785] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 2312.960703] env[63593]: INFO nova.scheduler.client.report [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted allocations for instance 692af003-f18e-4cee-99fd-ef42d6e4b390 [ 2313.260626] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 2313.260789] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2313.260951] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2313.261121] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2313.261274] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2313.261446] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 2313.471393] env[63593]: DEBUG oslo_concurrency.lockutils [None req-cc17020b-5d37-4e6a-a71b-a048f9ddc932 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "692af003-f18e-4cee-99fd-ef42d6e4b390" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 267.121s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2313.471681] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "692af003-f18e-4cee-99fd-ef42d6e4b390" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 71.666s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2313.471907] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "692af003-f18e-4cee-99fd-ef42d6e4b390-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2313.472130] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "692af003-f18e-4cee-99fd-ef42d6e4b390-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2313.472319] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "692af003-f18e-4cee-99fd-ef42d6e4b390-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2313.475594] env[63593]: INFO nova.compute.manager [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Terminating instance [ 2313.477335] env[63593]: DEBUG nova.compute.manager [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2313.477521] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2313.477791] env[63593]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c2b5bed2-5586-480f-915d-1552a6f35514 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2313.486071] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae777f33-6cba-4170-8519-3a7d44a0a286 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2313.506280] env[63593]: WARNING nova.virt.vmwareapi.vmops [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 692af003-f18e-4cee-99fd-ef42d6e4b390 could not be found. [ 2313.506452] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2313.506625] env[63593]: INFO nova.compute.manager [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Took 0.03 seconds to destroy the instance on the hypervisor. [ 2313.506947] env[63593]: DEBUG oslo.service.loopingcall [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2313.507244] env[63593]: DEBUG nova.compute.manager [-] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2313.507390] env[63593]: DEBUG nova.network.neutron [-] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2314.013554] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2314.024145] env[63593]: DEBUG nova.network.neutron [-] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2314.527216] env[63593]: INFO nova.compute.manager [-] [instance: 692af003-f18e-4cee-99fd-ef42d6e4b390] Took 1.02 seconds to deallocate network for instance. [ 2315.012550] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2315.063117] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "c6fdc813-6c9f-4253-9298-bbedf71d9758" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2315.063412] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "c6fdc813-6c9f-4253-9298-bbedf71d9758" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2315.553100] env[63593]: DEBUG oslo_concurrency.lockutils [None req-0a022cd7-72be-46c9-8d77-6e2e8f2163ac tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "692af003-f18e-4cee-99fd-ef42d6e4b390" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.081s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2315.565621] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 2316.088526] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2316.088838] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2316.090251] env[63593]: INFO nova.compute.claims [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2317.127688] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-faccaf9f-efa5-4457-982f-a691b1c28700 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2317.135548] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fbd712e-a9be-4653-9406-4627c080b133 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2317.164363] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fae731b0-4662-416b-a12e-ca2e0b216f6a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2317.171618] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e4291ea-93c4-4f1e-a0b5-246a98615965 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2317.183996] env[63593]: DEBUG nova.compute.provider_tree [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2317.687184] env[63593]: DEBUG nova.scheduler.client.report [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2318.193260] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.104s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2318.193811] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 2318.699083] env[63593]: DEBUG nova.compute.utils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2318.700287] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 2318.700452] env[63593]: DEBUG nova.network.neutron [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2318.751787] env[63593]: DEBUG nova.policy [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c0a03b9520a349cb99065ce54ec21927', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '87f6487499be40a9b8bf821e1bd4de80', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 2319.003050] env[63593]: DEBUG nova.network.neutron [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Successfully created port: 1f7d4661-4d72-4519-bf37-b894d55962cc {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2319.007591] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2319.204252] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 2320.214374] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 2320.238556] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2320.238789] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2320.238945] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2320.239139] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2320.239284] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2320.239428] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2320.239635] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2320.239796] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2320.239958] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2320.240179] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2320.240311] env[63593]: DEBUG nova.virt.hardware [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2320.241187] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5aeb8bff-4e42-4001-9e70-093a8e42637a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2320.248794] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d82ba201-de8e-4b3e-a49e-1b3c23ca1272 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2320.387841] env[63593]: DEBUG nova.compute.manager [req-1bb5a455-36a2-4f4d-a529-ae6d8f56c982 req-0d505b53-6799-454b-b7fa-3111ed7c7585 service nova] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Received event network-vif-plugged-1f7d4661-4d72-4519-bf37-b894d55962cc {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 2320.388117] env[63593]: DEBUG oslo_concurrency.lockutils [req-1bb5a455-36a2-4f4d-a529-ae6d8f56c982 req-0d505b53-6799-454b-b7fa-3111ed7c7585 service nova] Acquiring lock "c6fdc813-6c9f-4253-9298-bbedf71d9758-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2320.388328] env[63593]: DEBUG oslo_concurrency.lockutils [req-1bb5a455-36a2-4f4d-a529-ae6d8f56c982 req-0d505b53-6799-454b-b7fa-3111ed7c7585 service nova] Lock "c6fdc813-6c9f-4253-9298-bbedf71d9758-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2320.388492] env[63593]: DEBUG oslo_concurrency.lockutils [req-1bb5a455-36a2-4f4d-a529-ae6d8f56c982 req-0d505b53-6799-454b-b7fa-3111ed7c7585 service nova] Lock "c6fdc813-6c9f-4253-9298-bbedf71d9758-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2320.388657] env[63593]: DEBUG nova.compute.manager [req-1bb5a455-36a2-4f4d-a529-ae6d8f56c982 req-0d505b53-6799-454b-b7fa-3111ed7c7585 service nova] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] No waiting events found dispatching network-vif-plugged-1f7d4661-4d72-4519-bf37-b894d55962cc {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2320.388908] env[63593]: WARNING nova.compute.manager [req-1bb5a455-36a2-4f4d-a529-ae6d8f56c982 req-0d505b53-6799-454b-b7fa-3111ed7c7585 service nova] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Received unexpected event network-vif-plugged-1f7d4661-4d72-4519-bf37-b894d55962cc for instance with vm_state building and task_state spawning. [ 2320.931501] env[63593]: DEBUG nova.network.neutron [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Successfully updated port: 1f7d4661-4d72-4519-bf37-b894d55962cc {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2320.957051] env[63593]: DEBUG nova.compute.manager [req-0afd6b9f-8609-4928-adf1-afcb581b45eb req-a7bab0ce-4d20-47aa-82d3-21f252d9d960 service nova] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Received event network-changed-1f7d4661-4d72-4519-bf37-b894d55962cc {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 2320.957051] env[63593]: DEBUG nova.compute.manager [req-0afd6b9f-8609-4928-adf1-afcb581b45eb req-a7bab0ce-4d20-47aa-82d3-21f252d9d960 service nova] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Refreshing instance network info cache due to event network-changed-1f7d4661-4d72-4519-bf37-b894d55962cc. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 2320.957051] env[63593]: DEBUG oslo_concurrency.lockutils [req-0afd6b9f-8609-4928-adf1-afcb581b45eb req-a7bab0ce-4d20-47aa-82d3-21f252d9d960 service nova] Acquiring lock "refresh_cache-c6fdc813-6c9f-4253-9298-bbedf71d9758" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2320.957051] env[63593]: DEBUG oslo_concurrency.lockutils [req-0afd6b9f-8609-4928-adf1-afcb581b45eb req-a7bab0ce-4d20-47aa-82d3-21f252d9d960 service nova] Acquired lock "refresh_cache-c6fdc813-6c9f-4253-9298-bbedf71d9758" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2320.957051] env[63593]: DEBUG nova.network.neutron [req-0afd6b9f-8609-4928-adf1-afcb581b45eb req-a7bab0ce-4d20-47aa-82d3-21f252d9d960 service nova] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Refreshing network info cache for port 1f7d4661-4d72-4519-bf37-b894d55962cc {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2321.433996] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "refresh_cache-c6fdc813-6c9f-4253-9298-bbedf71d9758" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2321.487778] env[63593]: DEBUG nova.network.neutron [req-0afd6b9f-8609-4928-adf1-afcb581b45eb req-a7bab0ce-4d20-47aa-82d3-21f252d9d960 service nova] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2321.557486] env[63593]: DEBUG nova.network.neutron [req-0afd6b9f-8609-4928-adf1-afcb581b45eb req-a7bab0ce-4d20-47aa-82d3-21f252d9d960 service nova] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2322.012122] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2322.060278] env[63593]: DEBUG oslo_concurrency.lockutils [req-0afd6b9f-8609-4928-adf1-afcb581b45eb req-a7bab0ce-4d20-47aa-82d3-21f252d9d960 service nova] Releasing lock "refresh_cache-c6fdc813-6c9f-4253-9298-bbedf71d9758" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2322.060635] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "refresh_cache-c6fdc813-6c9f-4253-9298-bbedf71d9758" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2322.060798] env[63593]: DEBUG nova.network.neutron [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2322.515269] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2322.515672] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2322.515722] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2322.515871] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2322.516766] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66f00d0e-1151-43aa-9366-b107dcca504a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2322.525072] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5cf68734-8e51-4fd4-b0d8-c490de5c8a9b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2322.538221] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdccbff4-c23f-4a22-8868-3451436c094d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2322.544065] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b7b14c2-63ef-4845-895f-607a2d19c2d0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2322.573144] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181487MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2322.573322] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2322.573473] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2322.599603] env[63593]: DEBUG nova.network.neutron [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2322.718703] env[63593]: DEBUG nova.network.neutron [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Updating instance_info_cache with network_info: [{"id": "1f7d4661-4d72-4519-bf37-b894d55962cc", "address": "fa:16:3e:51:ed:53", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1f7d4661-4d", "ovs_interfaceid": "1f7d4661-4d72-4519-bf37-b894d55962cc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2323.220885] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "refresh_cache-c6fdc813-6c9f-4253-9298-bbedf71d9758" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2323.221227] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Instance network_info: |[{"id": "1f7d4661-4d72-4519-bf37-b894d55962cc", "address": "fa:16:3e:51:ed:53", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1f7d4661-4d", "ovs_interfaceid": "1f7d4661-4d72-4519-bf37-b894d55962cc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 2323.221628] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:51:ed:53', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a15de394-0367-4921-a5c1-6ac8615e3283', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '1f7d4661-4d72-4519-bf37-b894d55962cc', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2323.229187] env[63593]: DEBUG oslo.service.loopingcall [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2323.229385] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2323.229596] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-973a2996-99d1-4092-94cd-b5aa90057d87 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2323.249095] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2323.249095] env[63593]: value = "task-1368119" [ 2323.249095] env[63593]: _type = "Task" [ 2323.249095] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2323.255938] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368119, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2323.597236] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance c6fdc813-6c9f-4253-9298-bbedf71d9758 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2323.597487] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 1 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2323.597580] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=704MB phys_disk=200GB used_disk=1GB total_vcpus=48 used_vcpus=1 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2323.624863] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2816ca78-741f-4534-b835-2c01eb4688f0 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2323.631839] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a84e1a7-a365-4fc7-82af-9dbbd4f8f90b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2323.660039] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d63704d-9bf8-449c-8b42-c17b376ba360 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2323.666498] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c994e360-00b8-412d-b6cd-008c29f4b962 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2323.678656] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2323.757983] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368119, 'name': CreateVM_Task, 'duration_secs': 0.297667} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2323.758164] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2323.765035] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2323.765206] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2323.765543] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2323.765774] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8f7b292f-e3cd-4b94-872f-2b85d384a25a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2323.769727] env[63593]: DEBUG oslo_vmware.api [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2323.769727] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52ea6bf0-fdd8-06a9-2280-b687011f8047" [ 2323.769727] env[63593]: _type = "Task" [ 2323.769727] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2323.777031] env[63593]: DEBUG oslo_vmware.api [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52ea6bf0-fdd8-06a9-2280-b687011f8047, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2324.181413] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2324.279597] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2324.279864] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2324.280103] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2324.280250] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2324.280427] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2324.280663] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-2b90026a-a12c-4e9e-9e07-732d244775cd {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.296952] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2324.297149] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2324.297809] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-1bfde565-3f3b-4101-8d30-0c670133956e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.302753] env[63593]: DEBUG oslo_vmware.api [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2324.302753] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52917cb3-7774-a2ed-be43-844e19a75708" [ 2324.302753] env[63593]: _type = "Task" [ 2324.302753] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2324.309474] env[63593]: DEBUG oslo_vmware.api [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52917cb3-7774-a2ed-be43-844e19a75708, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2324.686769] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2324.687237] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.113s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2324.814600] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2324.814808] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] vmware_temp/f9869520-60cd-4a62-8527-3a750c1e6ad9/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2324.815059] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bcab5d67-077e-4369-871f-b29bf3c4862c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.833792] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] vmware_temp/f9869520-60cd-4a62-8527-3a750c1e6ad9/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2324.833985] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Fetch image to [datastore2] vmware_temp/f9869520-60cd-4a62-8527-3a750c1e6ad9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2324.834168] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/f9869520-60cd-4a62-8527-3a750c1e6ad9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2324.834937] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6159342f-c11f-4a2a-9a20-00d431e756d7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.841266] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6989997-a9f8-4401-a8bc-f21896c45c2e {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.849866] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14df2654-8f30-4065-a16d-e2425fc6d2ac {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.879010] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a6a22ee-6f5c-40dd-a7f8-6baad99eb65b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.884217] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4cbc9a77-3684-4e9b-80b9-7b0d05b8b293 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2324.903593] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2324.947361] env[63593]: DEBUG oslo_vmware.rw_handles [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f9869520-60cd-4a62-8527-3a750c1e6ad9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2325.007713] env[63593]: DEBUG oslo_vmware.rw_handles [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2325.007911] env[63593]: DEBUG oslo_vmware.rw_handles [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f9869520-60cd-4a62-8527-3a750c1e6ad9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2372.687669] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2372.688017] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2372.688100] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2372.688241] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=63593) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 2372.987064] env[63593]: WARNING oslo_vmware.rw_handles [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles response.begin() [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2372.987064] env[63593]: ERROR oslo_vmware.rw_handles [ 2372.987595] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Downloaded image file data 102d556e-9db2-4e01-84ad-6f8305934950 to vmware_temp/f9869520-60cd-4a62-8527-3a750c1e6ad9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2372.989609] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Caching image {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2372.989863] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Copying Virtual Disk [datastore2] vmware_temp/f9869520-60cd-4a62-8527-3a750c1e6ad9/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk to [datastore2] vmware_temp/f9869520-60cd-4a62-8527-3a750c1e6ad9/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk {{(pid=63593) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2372.990158] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-2e7956af-31fc-4bc9-a70c-e8fb9232d9f7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2372.999522] env[63593]: DEBUG oslo_vmware.api [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2372.999522] env[63593]: value = "task-1368120" [ 2372.999522] env[63593]: _type = "Task" [ 2372.999522] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2373.007082] env[63593]: DEBUG oslo_vmware.api [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368120, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2373.012543] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2373.012694] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Starting heal instance info cache {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 2373.012810] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Rebuilding the list of instances to heal {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 2373.509173] env[63593]: DEBUG oslo_vmware.exceptions [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Fault InvalidArgument not matched. {{(pid=63593) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2373.509488] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2373.510049] env[63593]: ERROR nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2373.510049] env[63593]: Faults: ['InvalidArgument'] [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Traceback (most recent call last): [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] yield resources [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] self.driver.spawn(context, instance, image_meta, [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] self._fetch_image_if_missing(context, vi) [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] image_cache(vi, tmp_image_ds_loc) [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] vm_util.copy_virtual_disk( [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] session._wait_for_task(vmdk_copy_task) [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] return self.wait_for_task(task_ref) [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] return evt.wait() [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] result = hub.switch() [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] return self.greenlet.switch() [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] self.f(*self.args, **self.kw) [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] raise exceptions.translate_fault(task_info.error) [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Faults: ['InvalidArgument'] [ 2373.510049] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] [ 2373.511347] env[63593]: INFO nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Terminating instance [ 2373.513407] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Start destroying the instance on the hypervisor. {{(pid=63593) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 2373.513641] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Destroying instance {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2373.514442] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6878935-d85b-4e0a-be1e-69d607a4d621 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2373.517519] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Skipping network cache update for instance because it is Building. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 2373.517665] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Didn't find any instances for network info cache update. {{(pid=63593) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 2373.517857] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2373.521843] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Unregistering the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2373.522081] env[63593]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-049ad638-143c-4d8f-a27a-12c9730b9336 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2373.584834] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Unregistered the VM {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2373.585086] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Deleting contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2373.585285] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleting the datastore file [datastore2] c6fdc813-6c9f-4253-9298-bbedf71d9758 {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2373.585532] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-e33c7ca0-fce2-46a7-b268-bad19d10afa6 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2373.592372] env[63593]: DEBUG oslo_vmware.api [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2373.592372] env[63593]: value = "task-1368122" [ 2373.592372] env[63593]: _type = "Task" [ 2373.592372] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2373.599423] env[63593]: DEBUG oslo_vmware.api [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368122, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2374.013591] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2374.102475] env[63593]: DEBUG oslo_vmware.api [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': task-1368122, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.070452} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2374.102720] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted the datastore file {{(pid=63593) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2374.102898] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Deleted contents of the VM from datastore datastore2 {{(pid=63593) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2374.103080] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Instance destroyed {{(pid=63593) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2374.103254] env[63593]: INFO nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Took 0.59 seconds to destroy the instance on the hypervisor. [ 2374.105294] env[63593]: DEBUG nova.compute.claims [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Aborting claim: {{(pid=63593) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 2374.105477] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2374.105710] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2374.666657] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5a7b44d-3378-4d4a-8b66-ff26a32c4a9a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2374.673889] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0735c3aa-d9be-4b87-8551-8c2726e65d6d {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2374.702448] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1814cd9e-1bd7-43e4-a3c9-daf0f7c25b56 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2374.708973] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f254e5b-8463-4031-aeca-2a8979426a7b {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2374.722525] env[63593]: DEBUG nova.compute.provider_tree [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2375.225288] env[63593]: DEBUG nova.scheduler.client.report [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2375.730417] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.624s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2375.730975] env[63593]: ERROR nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2375.730975] env[63593]: Faults: ['InvalidArgument'] [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Traceback (most recent call last): [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] self.driver.spawn(context, instance, image_meta, [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] self._fetch_image_if_missing(context, vi) [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] image_cache(vi, tmp_image_ds_loc) [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] vm_util.copy_virtual_disk( [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] session._wait_for_task(vmdk_copy_task) [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] return self.wait_for_task(task_ref) [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] return evt.wait() [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] result = hub.switch() [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] return self.greenlet.switch() [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] self.f(*self.args, **self.kw) [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] raise exceptions.translate_fault(task_info.error) [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Faults: ['InvalidArgument'] [ 2375.730975] env[63593]: ERROR nova.compute.manager [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] [ 2375.731932] env[63593]: DEBUG nova.compute.utils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] VimFaultException {{(pid=63593) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2375.733781] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Build of instance c6fdc813-6c9f-4253-9298-bbedf71d9758 was re-scheduled: A specified parameter was not correct: fileType [ 2375.733781] env[63593]: Faults: ['InvalidArgument'] {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 2375.734176] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Unplugging VIFs for instance {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 2375.734350] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=63593) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 2375.734515] env[63593]: DEBUG nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Deallocating network for instance {{(pid=63593) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 2375.734673] env[63593]: DEBUG nova.network.neutron [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] deallocate_for_instance() {{(pid=63593) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2376.471984] env[63593]: DEBUG nova.network.neutron [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Updating instance_info_cache with network_info: [] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2376.975124] env[63593]: INFO nova.compute.manager [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: c6fdc813-6c9f-4253-9298-bbedf71d9758] Took 1.24 seconds to deallocate network for instance. [ 2377.012158] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2378.011226] env[63593]: INFO nova.scheduler.client.report [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Deleted allocations for instance c6fdc813-6c9f-4253-9298-bbedf71d9758 [ 2378.519331] env[63593]: DEBUG oslo_concurrency.lockutils [None req-b2e9b3fa-a761-4b8f-b085-19c592c5812a tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "c6fdc813-6c9f-4253-9298-bbedf71d9758" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 63.456s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2379.357394] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "231fb251-1442-4e1d-a659-4bd580850f65" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2379.357765] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "231fb251-1442-4e1d-a659-4bd580850f65" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2379.859930] env[63593]: DEBUG nova.compute.manager [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Starting instance... {{(pid=63593) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 2380.012923] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2380.378315] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2380.378598] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2380.379930] env[63593]: INFO nova.compute.claims [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2381.405062] env[63593]: DEBUG nova.scheduler.client.report [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Refreshing inventories for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 2381.417283] env[63593]: DEBUG nova.scheduler.client.report [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Updating ProviderTree inventory for provider 7345f17f-3af2-4b0a-9521-0292dc691877 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 2381.417481] env[63593]: DEBUG nova.compute.provider_tree [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Updating inventory in ProviderTree for provider 7345f17f-3af2-4b0a-9521-0292dc691877 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2381.427583] env[63593]: DEBUG nova.scheduler.client.report [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Refreshing aggregate associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, aggregates: None {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 2381.443330] env[63593]: DEBUG nova.scheduler.client.report [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Refreshing trait associations for resource provider 7345f17f-3af2-4b0a-9521-0292dc691877, traits: COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_NODE {{(pid=63593) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 2381.463994] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af8c5c3f-cee2-4ed7-976d-a58157ee8470 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2381.471647] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f078d07-3756-4c1c-89bb-4cee3b177c36 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2381.500273] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-385302ef-e140-4169-a4f4-275f68812314 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2381.506906] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d04745c-165a-49e1-9f6a-4cfca8caab88 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2381.510534] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2381.519711] env[63593]: DEBUG nova.compute.provider_tree [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2382.022789] env[63593]: DEBUG nova.scheduler.client.report [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2382.527647] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.149s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2382.528182] env[63593]: DEBUG nova.compute.manager [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Start building networks asynchronously for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 2383.007969] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2383.032450] env[63593]: DEBUG nova.compute.utils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Using /dev/sd instead of None {{(pid=63593) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2383.033705] env[63593]: DEBUG nova.compute.manager [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Allocating IP information in the background. {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 2383.033849] env[63593]: DEBUG nova.network.neutron [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] allocate_for_instance() {{(pid=63593) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2383.085592] env[63593]: DEBUG nova.policy [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c0a03b9520a349cb99065ce54ec21927', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '87f6487499be40a9b8bf821e1bd4de80', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=63593) authorize /opt/stack/nova/nova/policy.py:203}} [ 2383.327824] env[63593]: DEBUG nova.network.neutron [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Successfully created port: b7be33c7-4179-4f40-9054-b649992b37fb {{(pid=63593) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2383.513255] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager.update_available_resource {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2383.539817] env[63593]: DEBUG nova.compute.manager [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Start building block device mappings for instance. {{(pid=63593) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 2384.018964] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2384.019225] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2384.019399] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2384.019556] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=63593) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2384.020480] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78a289eb-a22f-4b41-8c31-2f9a9e348536 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2384.028569] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1356c66c-f86b-4b5f-8817-ed863d624fa4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2384.041665] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7591c653-4843-40b3-afdf-f94f1402bda7 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2384.050485] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42991a9f-4594-4e55-b5d7-000af56c6623 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2384.080127] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181453MB free_disk=148GB free_vcpus=48 pci_devices=None {{(pid=63593) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2384.080294] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2384.080484] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2384.551325] env[63593]: DEBUG nova.compute.manager [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Start spawning the instance on the hypervisor. {{(pid=63593) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 2384.577081] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-10-26T05:52:18Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-10-26T05:52:01Z,direct_url=,disk_format='vmdk',id=102d556e-9db2-4e01-84ad-6f8305934950,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='48f00349e59f41fdb776b74d63555cac',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-10-26T05:52:01Z,virtual_size=,visibility=), allow threads: False {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2384.577325] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2384.577505] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image limits 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2384.577805] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Flavor pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2384.577878] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Image pref 0:0:0 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2384.577989] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=63593) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2384.578215] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2384.578372] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2384.578537] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Got 1 possible topologies {{(pid=63593) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2384.578698] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2384.578865] env[63593]: DEBUG nova.virt.hardware [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=63593) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2384.579762] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91250c4f-6b18-4c39-bc06-8f5f89901a82 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2384.588493] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4420ac27-a6b7-46b7-b705-ee66caac7c24 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2384.712500] env[63593]: DEBUG nova.compute.manager [req-e4c8c268-fbb6-41e6-82d8-f51be4f65ed9 req-dcd9879a-f6e2-42e7-9901-ec3f51d44be4 service nova] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Received event network-vif-plugged-b7be33c7-4179-4f40-9054-b649992b37fb {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 2384.712775] env[63593]: DEBUG oslo_concurrency.lockutils [req-e4c8c268-fbb6-41e6-82d8-f51be4f65ed9 req-dcd9879a-f6e2-42e7-9901-ec3f51d44be4 service nova] Acquiring lock "231fb251-1442-4e1d-a659-4bd580850f65-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2384.712960] env[63593]: DEBUG oslo_concurrency.lockutils [req-e4c8c268-fbb6-41e6-82d8-f51be4f65ed9 req-dcd9879a-f6e2-42e7-9901-ec3f51d44be4 service nova] Lock "231fb251-1442-4e1d-a659-4bd580850f65-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2384.713105] env[63593]: DEBUG oslo_concurrency.lockutils [req-e4c8c268-fbb6-41e6-82d8-f51be4f65ed9 req-dcd9879a-f6e2-42e7-9901-ec3f51d44be4 service nova] Lock "231fb251-1442-4e1d-a659-4bd580850f65-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2384.713272] env[63593]: DEBUG nova.compute.manager [req-e4c8c268-fbb6-41e6-82d8-f51be4f65ed9 req-dcd9879a-f6e2-42e7-9901-ec3f51d44be4 service nova] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] No waiting events found dispatching network-vif-plugged-b7be33c7-4179-4f40-9054-b649992b37fb {{(pid=63593) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2384.713436] env[63593]: WARNING nova.compute.manager [req-e4c8c268-fbb6-41e6-82d8-f51be4f65ed9 req-dcd9879a-f6e2-42e7-9901-ec3f51d44be4 service nova] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Received unexpected event network-vif-plugged-b7be33c7-4179-4f40-9054-b649992b37fb for instance with vm_state building and task_state spawning. [ 2384.793841] env[63593]: DEBUG nova.network.neutron [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Successfully updated port: b7be33c7-4179-4f40-9054-b649992b37fb {{(pid=63593) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2385.105774] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Instance 231fb251-1442-4e1d-a659-4bd580850f65 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=63593) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2385.105978] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Total usable vcpus: 48, total allocated vcpus: 1 {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2385.106139] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=704MB phys_disk=200GB used_disk=1GB total_vcpus=48 used_vcpus=1 pci_stats=[] {{(pid=63593) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2385.207747] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-905cdaff-147e-4bb9-8e60-9539c50586d4 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2385.215448] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-edf558c3-444b-4b85-a641-a3c2eb96ef33 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2385.244159] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4a256d2-0574-4bcf-b7d5-96769ba801e1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2385.250944] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a210683-4899-4972-8be4-adc627d3e12c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2385.263680] env[63593]: DEBUG nova.compute.provider_tree [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed in ProviderTree for provider: 7345f17f-3af2-4b0a-9521-0292dc691877 {{(pid=63593) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2385.296620] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "refresh_cache-231fb251-1442-4e1d-a659-4bd580850f65" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2385.296751] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "refresh_cache-231fb251-1442-4e1d-a659-4bd580850f65" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2385.296938] env[63593]: DEBUG nova.network.neutron [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Building network info cache for instance {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2385.767613] env[63593]: DEBUG nova.scheduler.client.report [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Inventory has not changed for provider 7345f17f-3af2-4b0a-9521-0292dc691877 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 148, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=63593) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2385.827994] env[63593]: DEBUG nova.network.neutron [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Instance cache missing network info. {{(pid=63593) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2385.948419] env[63593]: DEBUG nova.network.neutron [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Updating instance_info_cache with network_info: [{"id": "b7be33c7-4179-4f40-9054-b649992b37fb", "address": "fa:16:3e:66:66:82", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb7be33c7-41", "ovs_interfaceid": "b7be33c7-4179-4f40-9054-b649992b37fb", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2386.273206] env[63593]: DEBUG nova.compute.resource_tracker [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=63593) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2386.273448] env[63593]: DEBUG oslo_concurrency.lockutils [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.193s {{(pid=63593) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2386.273653] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2386.273785] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11226}} [ 2386.450746] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "refresh_cache-231fb251-1442-4e1d-a659-4bd580850f65" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2386.451242] env[63593]: DEBUG nova.compute.manager [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Instance network_info: |[{"id": "b7be33c7-4179-4f40-9054-b649992b37fb", "address": "fa:16:3e:66:66:82", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb7be33c7-41", "ovs_interfaceid": "b7be33c7-4179-4f40-9054-b649992b37fb", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=63593) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1996}} [ 2386.451440] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:66:66:82', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a15de394-0367-4921-a5c1-6ac8615e3283', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'b7be33c7-4179-4f40-9054-b649992b37fb', 'vif_model': 'vmxnet3'}] {{(pid=63593) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2386.458804] env[63593]: DEBUG oslo.service.loopingcall [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=63593) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2386.459013] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Creating VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2386.459242] env[63593]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-8aabc406-522f-4760-94d9-da12fb47fe06 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2386.479357] env[63593]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2386.479357] env[63593]: value = "task-1368123" [ 2386.479357] env[63593]: _type = "Task" [ 2386.479357] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2386.488390] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368123, 'name': CreateVM_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2386.739049] env[63593]: DEBUG nova.compute.manager [req-e4cbd973-4b89-4ec4-8edb-5696bdeae037 req-68398df4-980c-44d0-917a-d4200a96c0e1 service nova] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Received event network-changed-b7be33c7-4179-4f40-9054-b649992b37fb {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 2386.739192] env[63593]: DEBUG nova.compute.manager [req-e4cbd973-4b89-4ec4-8edb-5696bdeae037 req-68398df4-980c-44d0-917a-d4200a96c0e1 service nova] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Refreshing instance network info cache due to event network-changed-b7be33c7-4179-4f40-9054-b649992b37fb. {{(pid=63593) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 2386.739481] env[63593]: DEBUG oslo_concurrency.lockutils [req-e4cbd973-4b89-4ec4-8edb-5696bdeae037 req-68398df4-980c-44d0-917a-d4200a96c0e1 service nova] Acquiring lock "refresh_cache-231fb251-1442-4e1d-a659-4bd580850f65" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2386.739592] env[63593]: DEBUG oslo_concurrency.lockutils [req-e4cbd973-4b89-4ec4-8edb-5696bdeae037 req-68398df4-980c-44d0-917a-d4200a96c0e1 service nova] Acquired lock "refresh_cache-231fb251-1442-4e1d-a659-4bd580850f65" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2386.739678] env[63593]: DEBUG nova.network.neutron [req-e4cbd973-4b89-4ec4-8edb-5696bdeae037 req-68398df4-980c-44d0-917a-d4200a96c0e1 service nova] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Refreshing network info cache for port b7be33c7-4179-4f40-9054-b649992b37fb {{(pid=63593) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2386.776193] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] There are 0 instances to clean {{(pid=63593) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11235}} [ 2386.990344] env[63593]: DEBUG oslo_vmware.api [-] Task: {'id': task-1368123, 'name': CreateVM_Task, 'duration_secs': 0.28364} completed successfully. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2386.990594] env[63593]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Created VM on the ESX host {{(pid=63593) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2386.991277] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2386.991482] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2386.991802] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2386.992079] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-9d962df5-ba1e-4df9-887d-09e26d814126 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2386.997320] env[63593]: DEBUG oslo_vmware.api [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2386.997320] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52e9d47b-3cb7-027d-b302-7669fdf82941" [ 2386.997320] env[63593]: _type = "Task" [ 2386.997320] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2387.006950] env[63593]: DEBUG oslo_vmware.api [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52e9d47b-3cb7-027d-b302-7669fdf82941, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2387.442439] env[63593]: DEBUG nova.network.neutron [req-e4cbd973-4b89-4ec4-8edb-5696bdeae037 req-68398df4-980c-44d0-917a-d4200a96c0e1 service nova] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Updated VIF entry in instance network info cache for port b7be33c7-4179-4f40-9054-b649992b37fb. {{(pid=63593) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2387.442809] env[63593]: DEBUG nova.network.neutron [req-e4cbd973-4b89-4ec4-8edb-5696bdeae037 req-68398df4-980c-44d0-917a-d4200a96c0e1 service nova] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Updating instance_info_cache with network_info: [{"id": "b7be33c7-4179-4f40-9054-b649992b37fb", "address": "fa:16:3e:66:66:82", "network": {"id": "4515d818-cd5a-440c-ace5-d4c9a76c5bfd", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-121121742-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "87f6487499be40a9b8bf821e1bd4de80", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a15de394-0367-4921-a5c1-6ac8615e3283", "external-id": "nsx-vlan-transportzone-13", "segmentation_id": 13, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb7be33c7-41", "ovs_interfaceid": "b7be33c7-4179-4f40-9054-b649992b37fb", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=63593) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2387.508163] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Releasing lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2387.508356] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Processing image 102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2387.508588] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2387.508733] env[63593]: DEBUG oslo_concurrency.lockutils [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Acquired lock "[datastore2] devstack-image-cache_base/102d556e-9db2-4e01-84ad-6f8305934950/102d556e-9db2-4e01-84ad-6f8305934950.vmdk" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2387.508908] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2387.509179] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-05a329b2-7a40-41ca-84d8-5a7e7f569c6a {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2387.528155] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2387.528371] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=63593) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2387.529081] env[63593]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-1fda8dc5-9957-4ed9-a4e9-87499424a580 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2387.534595] env[63593]: DEBUG oslo_vmware.api [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Waiting for the task: (returnval){ [ 2387.534595] env[63593]: value = "session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52d7be91-12b8-1f30-d7fd-fdff888dbbf4" [ 2387.534595] env[63593]: _type = "Task" [ 2387.534595] env[63593]: } to complete. {{(pid=63593) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2387.542784] env[63593]: DEBUG oslo_vmware.api [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Task: {'id': session[525f20c6-1c2e-43c8-c080-a4b43f2a33ff]52d7be91-12b8-1f30-d7fd-fdff888dbbf4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=63593) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2387.946041] env[63593]: DEBUG oslo_concurrency.lockutils [req-e4cbd973-4b89-4ec4-8edb-5696bdeae037 req-68398df4-980c-44d0-917a-d4200a96c0e1 service nova] Releasing lock "refresh_cache-231fb251-1442-4e1d-a659-4bd580850f65" {{(pid=63593) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2388.045098] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Preparing fetch location {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2388.045559] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating directory with path [datastore2] vmware_temp/c943f7f6-98dd-4758-a47c-36c057124c91/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2388.045704] env[63593]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-6c06a82a-b232-4638-b3db-6abb82f629ac {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2388.065488] env[63593]: DEBUG nova.virt.vmwareapi.ds_util [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Created directory with path [datastore2] vmware_temp/c943f7f6-98dd-4758-a47c-36c057124c91/102d556e-9db2-4e01-84ad-6f8305934950 {{(pid=63593) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2388.065737] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Fetch image to [datastore2] vmware_temp/c943f7f6-98dd-4758-a47c-36c057124c91/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk {{(pid=63593) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2388.065864] env[63593]: DEBUG nova.virt.vmwareapi.vmops [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to [datastore2] vmware_temp/c943f7f6-98dd-4758-a47c-36c057124c91/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk on the data store datastore2 {{(pid=63593) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2388.066653] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09606281-2baf-45ec-a154-0ce60dacdc4c {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2388.073882] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c547a56-3b74-4ae4-b168-cf456fa01f88 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2388.084551] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abbec085-d0a7-48df-9e34-f4b15ecfcff5 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2388.114840] env[63593]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9515c1c-a22b-4497-bf65-278f707f1efb {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2388.120770] env[63593]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-c8a92b41-e063-4407-9c5c-bf4998cca5d1 {{(pid=63593) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2388.142053] env[63593]: DEBUG nova.virt.vmwareapi.images [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] [instance: 231fb251-1442-4e1d-a659-4bd580850f65] Downloading image file data 102d556e-9db2-4e01-84ad-6f8305934950 to the data store datastore2 {{(pid=63593) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2388.193443] env[63593]: DEBUG oslo_vmware.rw_handles [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c943f7f6-98dd-4758-a47c-36c057124c91/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2388.255327] env[63593]: DEBUG oslo_vmware.rw_handles [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Completed reading data from the image iterator. {{(pid=63593) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2388.255520] env[63593]: DEBUG oslo_vmware.rw_handles [None req-19b4ba00-66d9-49bb-accc-35ea947c6c32 tempest-DeleteServersTestJSON-894220685 tempest-DeleteServersTestJSON-894220685-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/c943f7f6-98dd-4758-a47c-36c057124c91/102d556e-9db2-4e01-84ad-6f8305934950/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=63593) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2389.013047] env[63593]: DEBUG oslo_service.periodic_task [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=63593) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2389.013441] env[63593]: DEBUG nova.compute.manager [None req-fee2ee31-f24d-4f16-967c-2d4f9cd0a7a0 None None] Cleaning up deleted instances with incomplete migration {{(pid=63593) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11264}}